Buckets:
| import{s as Xa,o as Ja,n as Ga}from"../chunks/scheduler.9bc65507.js";import{S as Qa,i as Ya,g as s,s as a,r as m,A as Za,h as l,f as o,c as r,j as v,u as p,x as d,k as _,y as t,a as i,v as c,d as g,t as f,w as u}from"../chunks/index.707bf1b6.js";import{T as Ra}from"../chunks/Tip.c2ecdbf4.js";import{D as C}from"../chunks/Docstring.86474e80.js";import{H as A,E as er}from"../chunks/EditOnGithub.922df6ba.js";function tr(we){let h,q=`For best performance, this data collator should be used with a dataset having items that are dictionaries or | |
| BatchEncoding, with the <code>"special_tokens_mask"</code> key, as returned by a <a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or a | |
| <a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> with the argument <code>return_special_tokens_mask=True</code>.`;return{c(){h=s("p"),h.innerHTML=q},l($){h=l($,"P",{"data-svelte-h":!0}),d(h)!=="svelte-7d9wd8"&&(h.innerHTML=q)},m($,H){i($,h,H)},p:Ga,d($){$&&o(h)}}}function ar(we){let h,q=`This collator relies on details of the implementation of subword tokenization by <a href="/docs/transformers/pr_30862/ja/model_doc/bert#transformers.BertTokenizer">BertTokenizer</a>, specifically | |
| that subword tokens are prefixed with <em>##</em>. For tokenizers that do not adhere to this scheme, this collator will | |
| produce an output that is roughly equivalent to <code>.DataCollatorForLanguageModeling</code>.`;return{c(){h=s("p"),h.innerHTML=q},l($){h=l($,"P",{"data-svelte-h":!0}),d(h)!=="svelte-5y1tai"&&(h.innerHTML=q)},m($,H){i($,h,H)},p:Ga,d($){$&&o(h)}}}function rr(we){let h,q,$,H,G,at,X,_a=`データ照合器は、データセット要素のリストを入力として使用してバッチを形成するオブジェクトです。これらの要素は、 | |
| <code>train_dataset</code> または <code>eval_dataset</code> の要素と同じ型。`,rt,J,ba=`バッチを構築できるようにするために、データ照合者は何らかの処理 (パディングなど) を適用する場合があります。そのうちのいくつかは( | |
| <code>DataCollatorForLanguageModeling</code>) ランダムなデータ拡張 (ランダム マスキングなど) も適用します | |
| 形成されたバッチ上で。`,ot,Q,va='使用例は、<a href="../examples">サンプル スクリプト</a> または <a href="../notebooks">サンプル ノートブック</a> にあります。',nt,Y,st,y,Z,zt,Pe,$a=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for | |
| potential keys named:`,St,Fe,ka="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",jt,Le,xa=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs | |
| to the model. See glue and ner for example of how it’s useful.`,lt,ee,it,T,te,It,qe,Ca=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for | |
| potential keys named:`,Wt,Me,Ta="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",At,ze,Da=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs | |
| to the model. See glue and ner for example of how it’s useful.`,Ht,Se,ya=`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be | |
| helpful if you need to set a return_tensors value at initialization.`,dt,ae,mt,j,re,Vt,je,wa="Data collator that will dynamically pad the inputs received.",pt,oe,ct,I,ne,Et,Ie,Pa="Data collator that will dynamically pad the inputs received, as well as the labels.",gt,se,ft,W,le,Nt,We,Fa="Data collator that will dynamically pad the inputs received, as well as the labels.",ut,ie,ht,k,de,Ot,Ae,La=`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they | |
| are not all of the same length.`,Bt,V,Ut,E,me,Kt,He,qa="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",Rt,N,pe,Gt,Ve,Ma="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",Xt,O,ce,Jt,Ee,za="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",_t,ge,bt,b,fe,Qt,Ne,Sa="Data collator used for language modeling that masks entire words.",Yt,Oe,ja="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for masked language modeling</li>",Zt,B,ea,U,ue,ta,Be,Ia=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set | |
| ‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,aa,K,he,ra,Ue,Wa=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set | |
| ‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,oa,R,_e,na,Ke,Aa=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set | |
| ‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,vt,be,$t,x,ve,sa,Re,Ha="Data collator used for permutation language modeling.",la,Ge,Va="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for permutation language modeling with procedures specific to XLNet</li>",ia,M,$e,da,Xe,Ea="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",ma,ke,Na=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be | |
| masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len < max_len</code> (i.e. there are tokens remaining in the | |
| sequence to be processed), repeat from Step 1.</li>`,pa,z,xe,ca,Je,Oa="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",ga,Ce,Ba=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be | |
| masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len < max_len</code> (i.e. there are tokens remaining in the | |
| sequence to be processed), repeat from Step 1.</li>`,fa,S,Te,ua,Qe,Ua="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",ha,De,Ka=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be | |
| masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len < max_len</code> (i.e. there are tokens remaining in the | |
| sequence to be processed), repeat from Step 1.</li>`,kt,ye,xt,tt,Ct;return G=new A({props:{title:"データ照合者",local:"データ照合者",headingTag:"h1"}}),Y=new A({props:{title:"Default data collator",local:"transformers.default_data_collator",headingTag:"h2"}}),Z=new C({props:{name:"transformers.default_data_collator",anchor:"transformers.default_data_collator",parameters:[{name:"features",val:": List"},{name:"return_tensors",val:" = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L74"}}),ee=new A({props:{title:"DefaultDataCollator",local:"transformers.DefaultDataCollator",headingTag:"h2"}}),te=new C({props:{name:"class transformers.DefaultDataCollator",anchor:"transformers.DefaultDataCollator",parameters:[{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DefaultDataCollator.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"pt"</code>) — | |
| The type of Tensor to return. Allowable values are “np”, “pt” and “tf”.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L99"}}),ae=new A({props:{title:"DataCollatorWithPadding",local:"transformers.DataCollatorWithPadding",headingTag:"h2"}}),re=new C({props:{name:"class transformers.DataCollatorWithPadding",anchor:"transformers.DataCollatorWithPadding",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": Union = True"},{name:"max_length",val:": Optional = None"},{name:"pad_to_multiple_of",val:": Optional = None"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorWithPadding.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) — | |
| The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorWithPadding.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_30862/ja/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) — | |
| Select a strategy to pad the returned sequences (according to the model’s padding side and padding index) | |
| among:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest'</code> (default): Pad to the longest sequence in the batch (or no padding if only a single | |
| sequence is provided).</li> | |
| <li><code>'max_length'</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum | |
| acceptable input length for the model if that argument is not provided.</li> | |
| <li><code>False</code> or <code>'do_not_pad'</code>: No padding (i.e., can output a batch with sequences of different lengths).</li> | |
| </ul>`,name:"padding"},{anchor:"transformers.DataCollatorWithPadding.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorWithPadding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value.</p> | |
| <p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= | |
| 7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorWithPadding.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"pt"</code>) — | |
| The type of Tensor to return. Allowable values are “np”, “pt” and “tf”.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L236"}}),oe=new A({props:{title:"DataCollatorForTokenClassification",local:"transformers.DataCollatorForTokenClassification",headingTag:"h2"}}),ne=new C({props:{name:"class transformers.DataCollatorForTokenClassification",anchor:"transformers.DataCollatorForTokenClassification",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": Union = True"},{name:"max_length",val:": Optional = None"},{name:"pad_to_multiple_of",val:": Optional = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForTokenClassification.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) — | |
| The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForTokenClassification.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_30862/ja/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) — | |
| Select a strategy to pad the returned sequences (according to the model’s padding side and padding index) | |
| among:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest'</code> (default): Pad to the longest sequence in the batch (or no padding if only a single | |
| sequence is provided).</li> | |
| <li><code>'max_length'</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum | |
| acceptable input length for the model if that argument is not provided.</li> | |
| <li><code>False</code> or <code>'do_not_pad'</code>: No padding (i.e., can output a batch with sequences of different lengths).</li> | |
| </ul>`,name:"padding"},{anchor:"transformers.DataCollatorForTokenClassification.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForTokenClassification.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value.</p> | |
| <p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= | |
| 7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForTokenClassification.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) — | |
| The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForTokenClassification.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"pt"</code>) — | |
| The type of Tensor to return. Allowable values are “np”, “pt” and “tf”.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L288"}}),se=new A({props:{title:"DataCollatorForSeq2Seq",local:"transformers.DataCollatorForSeq2Seq",headingTag:"h2"}}),le=new C({props:{name:"class transformers.DataCollatorForSeq2Seq",anchor:"transformers.DataCollatorForSeq2Seq",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"model",val:": Optional = None"},{name:"padding",val:": Union = True"},{name:"max_length",val:": Optional = None"},{name:"pad_to_multiple_of",val:": Optional = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForSeq2Seq.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) — | |
| The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForSeq2Seq.model",description:`<strong>model</strong> (<a href="/docs/transformers/pr_30862/ja/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>, <em>optional</em>) — | |
| The model that is being trained. If set and has the <em>prepare_decoder_input_ids_from_labels</em>, use it to | |
| prepare the <em>decoder_input_ids</em></p> | |
| <p>This is useful when using <em>label_smoothing</em> to avoid calculating loss twice.`,name:"model"},{anchor:"transformers.DataCollatorForSeq2Seq.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_30862/ja/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) — | |
| Select a strategy to pad the returned sequences (according to the model’s padding side and padding index) | |
| among:</p> | |
| <ul> | |
| <li><code>True</code> or <code>'longest'</code> (default): Pad to the longest sequence in the batch (or no padding if only a single | |
| sequence is provided).</li> | |
| <li><code>'max_length'</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum | |
| acceptable input length for the model if that argument is not provided.</li> | |
| <li><code>False</code> or <code>'do_not_pad'</code>: No padding (i.e., can output a batch with sequences of different lengths).</li> | |
| </ul>`,name:"padding"},{anchor:"transformers.DataCollatorForSeq2Seq.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) — | |
| Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForSeq2Seq.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value.</p> | |
| <p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= | |
| 7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForSeq2Seq.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) — | |
| The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForSeq2Seq.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"pt"</code>) — | |
| The type of Tensor to return. Allowable values are “np”, “pt” and “tf”.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L542"}}),ie=new A({props:{title:"DataCollatorForLanguageModeling",local:"transformers.DataCollatorForLanguageModeling",headingTag:"h2"}}),de=new C({props:{name:"class transformers.DataCollatorForLanguageModeling",anchor:"transformers.DataCollatorForLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"pad_to_multiple_of",val:": Optional = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForLanguageModeling.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_30862/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) — | |
| The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm",description:`<strong>mlm</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to use masked language modeling. If set to <code>False</code>, the labels are the same as the inputs | |
| with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked | |
| tokens and the value to predict for the masked token.`,name:"mlm"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm_probability",description:`<strong>mlm_probability</strong> (<code>float</code>, <em>optional</em>, defaults to 0.15) — | |
| The probability with which to (randomly) mask tokens in the input, when <code>mlm</code> is set to <code>True</code>.`,name:"mlm_probability"},{anchor:"transformers.DataCollatorForLanguageModeling.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) — | |
| If set will pad the sequence to a multiple of the provided value.`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForLanguageModeling.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) — | |
| The type of Tensor to return. Allowable values are “np”, “pt” and “tf”.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L678"}}),V=new Ra({props:{$$slots:{default:[tr]},$$scope:{ctx:we}}}),me=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": Any"},{name:"special_tokens_mask",val:": Optional = None"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L884"}}),pe=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": Any"},{name:"vocab_size",val:""},{name:"mask_token_id",val:""},{name:"special_tokens_mask",val:": Optional = None"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L731"}}),ce=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": Any"},{name:"special_tokens_mask",val:": Optional = None"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L827"}}),ge=new A({props:{title:"DataCollatorForWholeWordMask",local:"transformers.DataCollatorForWholeWordMask",headingTag:"h2"}}),fe=new C({props:{name:"class transformers.DataCollatorForWholeWordMask",anchor:"transformers.DataCollatorForWholeWordMask",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"pad_to_multiple_of",val:": Optional = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L922"}}),B=new Ra({props:{$$slots:{default:[ar]},$$scope:{ctx:we}}}),ue=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.numpy_mask_tokens",parameters:[{name:"inputs",val:": Any"},{name:"mask_labels",val:": Any"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L1153"}}),he=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.tf_mask_tokens",parameters:[{name:"inputs",val:": Any"},{name:"mask_labels",val:": Any"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L1111"}}),_e=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.torch_mask_tokens",parameters:[{name:"inputs",val:": Any"},{name:"mask_labels",val:": Any"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L1071"}}),be=new A({props:{title:"DataCollatorForPermutationLanguageModeling",local:"transformers.DataCollatorForPermutationLanguageModeling",headingTag:"h2"}}),ve=new C({props:{name:"class transformers.DataCollatorForPermutationLanguageModeling",anchor:"transformers.DataCollatorForPermutationLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"plm_probability",val:": float = 0.16666666666666666"},{name:"max_span_length",val:": int = 5"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L1277"}}),$e=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": Any"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L1518"}}),xe=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": Any"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L1411"}}),Te=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": Any"}],source:"https://github.com/huggingface/transformers/blob/vr_30862/src/transformers/data/data_collator.py#L1312"}}),ye=new er({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/ja/main_classes/data_collator.md"}}),{c(){h=s("meta"),q=a(),$=s("p"),H=a(),m(G.$$.fragment),at=a(),X=s("p"),X.innerHTML=_a,rt=a(),J=s("p"),J.innerHTML=ba,ot=a(),Q=s("p"),Q.innerHTML=va,nt=a(),m(Y.$$.fragment),st=a(),y=s("div"),m(Z.$$.fragment),zt=a(),Pe=s("p"),Pe.textContent=$a,St=a(),Fe=s("ul"),Fe.innerHTML=ka,jt=a(),Le=s("p"),Le.textContent=xa,lt=a(),m(ee.$$.fragment),it=a(),T=s("div"),m(te.$$.fragment),It=a(),qe=s("p"),qe.textContent=Ca,Wt=a(),Me=s("ul"),Me.innerHTML=Ta,At=a(),ze=s("p"),ze.textContent=Da,Ht=a(),Se=s("p"),Se.textContent=ya,dt=a(),m(ae.$$.fragment),mt=a(),j=s("div"),m(re.$$.fragment),Vt=a(),je=s("p"),je.textContent=wa,pt=a(),m(oe.$$.fragment),ct=a(),I=s("div"),m(ne.$$.fragment),Et=a(),Ie=s("p"),Ie.textContent=Pa,gt=a(),m(se.$$.fragment),ft=a(),W=s("div"),m(le.$$.fragment),Nt=a(),We=s("p"),We.textContent=Fa,ut=a(),m(ie.$$.fragment),ht=a(),k=s("div"),m(de.$$.fragment),Ot=a(),Ae=s("p"),Ae.textContent=La,Bt=a(),m(V.$$.fragment),Ut=a(),E=s("div"),m(me.$$.fragment),Kt=a(),He=s("p"),He.textContent=qa,Rt=a(),N=s("div"),m(pe.$$.fragment),Gt=a(),Ve=s("p"),Ve.textContent=Ma,Xt=a(),O=s("div"),m(ce.$$.fragment),Jt=a(),Ee=s("p"),Ee.textContent=za,_t=a(),m(ge.$$.fragment),bt=a(),b=s("div"),m(fe.$$.fragment),Qt=a(),Ne=s("p"),Ne.textContent=Sa,Yt=a(),Oe=s("ul"),Oe.innerHTML=ja,Zt=a(),m(B.$$.fragment),ea=a(),U=s("div"),m(ue.$$.fragment),ta=a(),Be=s("p"),Be.textContent=Ia,aa=a(),K=s("div"),m(he.$$.fragment),ra=a(),Ue=s("p"),Ue.textContent=Wa,oa=a(),R=s("div"),m(_e.$$.fragment),na=a(),Ke=s("p"),Ke.textContent=Aa,vt=a(),m(be.$$.fragment),$t=a(),x=s("div"),m(ve.$$.fragment),sa=a(),Re=s("p"),Re.textContent=Ha,la=a(),Ge=s("ul"),Ge.innerHTML=Va,ia=a(),M=s("div"),m($e.$$.fragment),da=a(),Xe=s("p"),Xe.textContent=Ea,ma=a(),ke=s("ol"),ke.innerHTML=Na,pa=a(),z=s("div"),m(xe.$$.fragment),ca=a(),Je=s("p"),Je.textContent=Oa,ga=a(),Ce=s("ol"),Ce.innerHTML=Ba,fa=a(),S=s("div"),m(Te.$$.fragment),ua=a(),Qe=s("p"),Qe.textContent=Ua,ha=a(),De=s("ol"),De.innerHTML=Ka,kt=a(),m(ye.$$.fragment),xt=a(),tt=s("p"),this.h()},l(e){const n=Za("svelte-u9bgzb",document.head);h=l(n,"META",{name:!0,content:!0}),n.forEach(o),q=r(e),$=l(e,"P",{}),v($).forEach(o),H=r(e),p(G.$$.fragment,e),at=r(e),X=l(e,"P",{"data-svelte-h":!0}),d(X)!=="svelte-wq0mnv"&&(X.innerHTML=_a),rt=r(e),J=l(e,"P",{"data-svelte-h":!0}),d(J)!=="svelte-c5nfv2"&&(J.innerHTML=ba),ot=r(e),Q=l(e,"P",{"data-svelte-h":!0}),d(Q)!=="svelte-qqt368"&&(Q.innerHTML=va),nt=r(e),p(Y.$$.fragment,e),st=r(e),y=l(e,"DIV",{class:!0});var L=v(y);p(Z.$$.fragment,L),zt=r(L),Pe=l(L,"P",{"data-svelte-h":!0}),d(Pe)!=="svelte-1hmsgsg"&&(Pe.textContent=$a),St=r(L),Fe=l(L,"UL",{"data-svelte-h":!0}),d(Fe)!=="svelte-1pq8qks"&&(Fe.innerHTML=ka),jt=r(L),Le=l(L,"P",{"data-svelte-h":!0}),d(Le)!=="svelte-1vi1gug"&&(Le.textContent=xa),L.forEach(o),lt=r(e),p(ee.$$.fragment,e),it=r(e),T=l(e,"DIV",{class:!0});var w=v(T);p(te.$$.fragment,w),It=r(w),qe=l(w,"P",{"data-svelte-h":!0}),d(qe)!=="svelte-1hmsgsg"&&(qe.textContent=Ca),Wt=r(w),Me=l(w,"UL",{"data-svelte-h":!0}),d(Me)!=="svelte-1pq8qks"&&(Me.innerHTML=Ta),At=r(w),ze=l(w,"P",{"data-svelte-h":!0}),d(ze)!=="svelte-1vi1gug"&&(ze.textContent=Da),Ht=r(w),Se=l(w,"P",{"data-svelte-h":!0}),d(Se)!=="svelte-vfvbwr"&&(Se.textContent=ya),w.forEach(o),dt=r(e),p(ae.$$.fragment,e),mt=r(e),j=l(e,"DIV",{class:!0});var Tt=v(j);p(re.$$.fragment,Tt),Vt=r(Tt),je=l(Tt,"P",{"data-svelte-h":!0}),d(je)!=="svelte-1iebpai"&&(je.textContent=wa),Tt.forEach(o),pt=r(e),p(oe.$$.fragment,e),ct=r(e),I=l(e,"DIV",{class:!0});var Dt=v(I);p(ne.$$.fragment,Dt),Et=r(Dt),Ie=l(Dt,"P",{"data-svelte-h":!0}),d(Ie)!=="svelte-4uvw1w"&&(Ie.textContent=Pa),Dt.forEach(o),gt=r(e),p(se.$$.fragment,e),ft=r(e),W=l(e,"DIV",{class:!0});var yt=v(W);p(le.$$.fragment,yt),Nt=r(yt),We=l(yt,"P",{"data-svelte-h":!0}),d(We)!=="svelte-4uvw1w"&&(We.textContent=Fa),yt.forEach(o),ut=r(e),p(ie.$$.fragment,e),ht=r(e),k=l(e,"DIV",{class:!0});var P=v(k);p(de.$$.fragment,P),Ot=r(P),Ae=l(P,"P",{"data-svelte-h":!0}),d(Ae)!=="svelte-10km8jk"&&(Ae.textContent=La),Bt=r(P),p(V.$$.fragment,P),Ut=r(P),E=l(P,"DIV",{class:!0});var wt=v(E);p(me.$$.fragment,wt),Kt=r(wt),He=l(wt,"P",{"data-svelte-h":!0}),d(He)!=="svelte-iv4xqf"&&(He.textContent=qa),wt.forEach(o),Rt=r(P),N=l(P,"DIV",{class:!0});var Pt=v(N);p(pe.$$.fragment,Pt),Gt=r(Pt),Ve=l(Pt,"P",{"data-svelte-h":!0}),d(Ve)!=="svelte-iv4xqf"&&(Ve.textContent=Ma),Pt.forEach(o),Xt=r(P),O=l(P,"DIV",{class:!0});var Ft=v(O);p(ce.$$.fragment,Ft),Jt=r(Ft),Ee=l(Ft,"P",{"data-svelte-h":!0}),d(Ee)!=="svelte-iv4xqf"&&(Ee.textContent=za),Ft.forEach(o),P.forEach(o),_t=r(e),p(ge.$$.fragment,e),bt=r(e),b=l(e,"DIV",{class:!0});var D=v(b);p(fe.$$.fragment,D),Qt=r(D),Ne=l(D,"P",{"data-svelte-h":!0}),d(Ne)!=="svelte-sjqk55"&&(Ne.textContent=Sa),Yt=r(D),Oe=l(D,"UL",{"data-svelte-h":!0}),d(Oe)!=="svelte-d2ozxj"&&(Oe.innerHTML=ja),Zt=r(D),p(B.$$.fragment,D),ea=r(D),U=l(D,"DIV",{class:!0});var Lt=v(U);p(ue.$$.fragment,Lt),ta=r(Lt),Be=l(Lt,"P",{"data-svelte-h":!0}),d(Be)!=="svelte-10msjh1"&&(Be.textContent=Ia),Lt.forEach(o),aa=r(D),K=l(D,"DIV",{class:!0});var qt=v(K);p(he.$$.fragment,qt),ra=r(qt),Ue=l(qt,"P",{"data-svelte-h":!0}),d(Ue)!=="svelte-10msjh1"&&(Ue.textContent=Wa),qt.forEach(o),oa=r(D),R=l(D,"DIV",{class:!0});var Mt=v(R);p(_e.$$.fragment,Mt),na=r(Mt),Ke=l(Mt,"P",{"data-svelte-h":!0}),d(Ke)!=="svelte-10msjh1"&&(Ke.textContent=Aa),Mt.forEach(o),D.forEach(o),vt=r(e),p(be.$$.fragment,e),$t=r(e),x=l(e,"DIV",{class:!0});var F=v(x);p(ve.$$.fragment,F),sa=r(F),Re=l(F,"P",{"data-svelte-h":!0}),d(Re)!=="svelte-12inif1"&&(Re.textContent=Ha),la=r(F),Ge=l(F,"UL",{"data-svelte-h":!0}),d(Ge)!=="svelte-1t4qh7k"&&(Ge.innerHTML=Va),ia=r(F),M=l(F,"DIV",{class:!0});var Ye=v(M);p($e.$$.fragment,Ye),da=r(Ye),Xe=l(Ye,"P",{"data-svelte-h":!0}),d(Xe)!=="svelte-1u70phx"&&(Xe.textContent=Ea),ma=r(Ye),ke=l(Ye,"OL",{start:!0,"data-svelte-h":!0}),d(ke)!=="svelte-1r6jrbi"&&(ke.innerHTML=Na),Ye.forEach(o),pa=r(F),z=l(F,"DIV",{class:!0});var Ze=v(z);p(xe.$$.fragment,Ze),ca=r(Ze),Je=l(Ze,"P",{"data-svelte-h":!0}),d(Je)!=="svelte-1u70phx"&&(Je.textContent=Oa),ga=r(Ze),Ce=l(Ze,"OL",{start:!0,"data-svelte-h":!0}),d(Ce)!=="svelte-1r6jrbi"&&(Ce.innerHTML=Ba),Ze.forEach(o),fa=r(F),S=l(F,"DIV",{class:!0});var et=v(S);p(Te.$$.fragment,et),ua=r(et),Qe=l(et,"P",{"data-svelte-h":!0}),d(Qe)!=="svelte-1u70phx"&&(Qe.textContent=Ua),ha=r(et),De=l(et,"OL",{start:!0,"data-svelte-h":!0}),d(De)!=="svelte-1r6jrbi"&&(De.innerHTML=Ka),et.forEach(o),F.forEach(o),kt=r(e),p(ye.$$.fragment,e),xt=r(e),tt=l(e,"P",{}),v(tt).forEach(o),this.h()},h(){_(h,"name","hf:doc:metadata"),_(h,"content",or),_(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(ke,"start","0"),_(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Ce,"start","0"),_(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(De,"start","0"),_(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){t(document.head,h),i(e,q,n),i(e,$,n),i(e,H,n),c(G,e,n),i(e,at,n),i(e,X,n),i(e,rt,n),i(e,J,n),i(e,ot,n),i(e,Q,n),i(e,nt,n),c(Y,e,n),i(e,st,n),i(e,y,n),c(Z,y,null),t(y,zt),t(y,Pe),t(y,St),t(y,Fe),t(y,jt),t(y,Le),i(e,lt,n),c(ee,e,n),i(e,it,n),i(e,T,n),c(te,T,null),t(T,It),t(T,qe),t(T,Wt),t(T,Me),t(T,At),t(T,ze),t(T,Ht),t(T,Se),i(e,dt,n),c(ae,e,n),i(e,mt,n),i(e,j,n),c(re,j,null),t(j,Vt),t(j,je),i(e,pt,n),c(oe,e,n),i(e,ct,n),i(e,I,n),c(ne,I,null),t(I,Et),t(I,Ie),i(e,gt,n),c(se,e,n),i(e,ft,n),i(e,W,n),c(le,W,null),t(W,Nt),t(W,We),i(e,ut,n),c(ie,e,n),i(e,ht,n),i(e,k,n),c(de,k,null),t(k,Ot),t(k,Ae),t(k,Bt),c(V,k,null),t(k,Ut),t(k,E),c(me,E,null),t(E,Kt),t(E,He),t(k,Rt),t(k,N),c(pe,N,null),t(N,Gt),t(N,Ve),t(k,Xt),t(k,O),c(ce,O,null),t(O,Jt),t(O,Ee),i(e,_t,n),c(ge,e,n),i(e,bt,n),i(e,b,n),c(fe,b,null),t(b,Qt),t(b,Ne),t(b,Yt),t(b,Oe),t(b,Zt),c(B,b,null),t(b,ea),t(b,U),c(ue,U,null),t(U,ta),t(U,Be),t(b,aa),t(b,K),c(he,K,null),t(K,ra),t(K,Ue),t(b,oa),t(b,R),c(_e,R,null),t(R,na),t(R,Ke),i(e,vt,n),c(be,e,n),i(e,$t,n),i(e,x,n),c(ve,x,null),t(x,sa),t(x,Re),t(x,la),t(x,Ge),t(x,ia),t(x,M),c($e,M,null),t(M,da),t(M,Xe),t(M,ma),t(M,ke),t(x,pa),t(x,z),c(xe,z,null),t(z,ca),t(z,Je),t(z,ga),t(z,Ce),t(x,fa),t(x,S),c(Te,S,null),t(S,ua),t(S,Qe),t(S,ha),t(S,De),i(e,kt,n),c(ye,e,n),i(e,xt,n),i(e,tt,n),Ct=!0},p(e,[n]){const L={};n&2&&(L.$$scope={dirty:n,ctx:e}),V.$set(L);const w={};n&2&&(w.$$scope={dirty:n,ctx:e}),B.$set(w)},i(e){Ct||(g(G.$$.fragment,e),g(Y.$$.fragment,e),g(Z.$$.fragment,e),g(ee.$$.fragment,e),g(te.$$.fragment,e),g(ae.$$.fragment,e),g(re.$$.fragment,e),g(oe.$$.fragment,e),g(ne.$$.fragment,e),g(se.$$.fragment,e),g(le.$$.fragment,e),g(ie.$$.fragment,e),g(de.$$.fragment,e),g(V.$$.fragment,e),g(me.$$.fragment,e),g(pe.$$.fragment,e),g(ce.$$.fragment,e),g(ge.$$.fragment,e),g(fe.$$.fragment,e),g(B.$$.fragment,e),g(ue.$$.fragment,e),g(he.$$.fragment,e),g(_e.$$.fragment,e),g(be.$$.fragment,e),g(ve.$$.fragment,e),g($e.$$.fragment,e),g(xe.$$.fragment,e),g(Te.$$.fragment,e),g(ye.$$.fragment,e),Ct=!0)},o(e){f(G.$$.fragment,e),f(Y.$$.fragment,e),f(Z.$$.fragment,e),f(ee.$$.fragment,e),f(te.$$.fragment,e),f(ae.$$.fragment,e),f(re.$$.fragment,e),f(oe.$$.fragment,e),f(ne.$$.fragment,e),f(se.$$.fragment,e),f(le.$$.fragment,e),f(ie.$$.fragment,e),f(de.$$.fragment,e),f(V.$$.fragment,e),f(me.$$.fragment,e),f(pe.$$.fragment,e),f(ce.$$.fragment,e),f(ge.$$.fragment,e),f(fe.$$.fragment,e),f(B.$$.fragment,e),f(ue.$$.fragment,e),f(he.$$.fragment,e),f(_e.$$.fragment,e),f(be.$$.fragment,e),f(ve.$$.fragment,e),f($e.$$.fragment,e),f(xe.$$.fragment,e),f(Te.$$.fragment,e),f(ye.$$.fragment,e),Ct=!1},d(e){e&&(o(q),o($),o(H),o(at),o(X),o(rt),o(J),o(ot),o(Q),o(nt),o(st),o(y),o(lt),o(it),o(T),o(dt),o(mt),o(j),o(pt),o(ct),o(I),o(gt),o(ft),o(W),o(ut),o(ht),o(k),o(_t),o(bt),o(b),o(vt),o($t),o(x),o(kt),o(xt),o(tt)),o(h),u(G,e),u(Y,e),u(Z),u(ee,e),u(te),u(ae,e),u(re),u(oe,e),u(ne),u(se,e),u(le),u(ie,e),u(de),u(V),u(me),u(pe),u(ce),u(ge,e),u(fe),u(B),u(ue),u(he),u(_e),u(be,e),u(ve),u($e),u(xe),u(Te),u(ye,e)}}}const or='{"title":"データ照合者","local":"データ照合者","sections":[{"title":"Default data collator","local":"transformers.default_data_collator","sections":[],"depth":2},{"title":"DefaultDataCollator","local":"transformers.DefaultDataCollator","sections":[],"depth":2},{"title":"DataCollatorWithPadding","local":"transformers.DataCollatorWithPadding","sections":[],"depth":2},{"title":"DataCollatorForTokenClassification","local":"transformers.DataCollatorForTokenClassification","sections":[],"depth":2},{"title":"DataCollatorForSeq2Seq","local":"transformers.DataCollatorForSeq2Seq","sections":[],"depth":2},{"title":"DataCollatorForLanguageModeling","local":"transformers.DataCollatorForLanguageModeling","sections":[],"depth":2},{"title":"DataCollatorForWholeWordMask","local":"transformers.DataCollatorForWholeWordMask","sections":[],"depth":2},{"title":"DataCollatorForPermutationLanguageModeling","local":"transformers.DataCollatorForPermutationLanguageModeling","sections":[],"depth":2}],"depth":1}';function nr(we){return Ja(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pr extends Qa{constructor(h){super(),Ya(this,h,nr,rr,Xa,{})}}export{pr as component}; | |
Xet Storage Details
- Size:
- 39.2 kB
- Xet hash:
- 6c970a9cb84b99dd0c9853e66b94993aefcc7142b4b56e9aa1d2948384bc95e6
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.