transformers/pr_36839/ja/_app/immutable/nodes/32.bda9ecba.js

import{s as Xa,o as Ja,n as Ga}from"../chunks/scheduler.bdbef820.js";import{S as Qa,i as Ya,g as l,s as o,r as m,A as Za,h as s,f as a,c as r,j as k,u as p,x as d,k as _,y as t,a as i,v as c,d as g,t as f,w as u,m as eo,n as to}from"../chunks/index.33f81d56.js";import{T as Ra}from"../chunks/Tip.34194030.js";import{D as C}from"../chunks/Docstring.64554317.js";import{H as K,E as ao}from"../chunks/EditOnGithub.a9246e21.js";function oo(Me){let h,A=`For best performance, this data collator should be used with a dataset having items that are dictionaries or
BatchEncoding, with the <code>&quot;special_tokens_mask&quot;</code> key, as returned by a <a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or a
<a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> with the argument <code>return_special_tokens_mask=True</code>.`,y,w,j="<li><p>Default Behavior:</p> <ul><li><code>mask_replace_prob=0.8</code>, <code>random_replace_prob=0.1</code>.</li> <li>Expect 80% of masked tokens replaced with <code>[MASK]</code>, 10% replaced with random tokens, and 10% left unchanged.</li></ul></li> <li><p>All masked tokens replaced by <code>[MASK]</code>:</p> <ul><li><code>mask_replace_prob=1.0</code>, <code>random_replace_prob=0.0</code>.</li> <li>Expect all masked tokens to be replaced with <code>[MASK]</code>. No tokens are left unchanged or replaced with random tokens.</li></ul></li> <li><p>No <code>[MASK]</code> replacement, only random tokens:</p> <ul><li><code>mask_replace_prob=0.0</code>, <code>random_replace_prob=1.0</code>.</li> <li>Expect all masked tokens to be replaced with random tokens. No <code>[MASK]</code> replacements or unchanged tokens.</li></ul></li> <li><p>Balanced replacement:</p> <ul><li><code>mask_replace_prob=0.5</code>, <code>random_replace_prob=0.4</code>.</li> <li>Expect 50% of masked tokens replaced with <code>[MASK]</code>, 40% replaced with random tokens, and 10% left unchanged.</li></ul></li>",W,T,ot=`Note:
The sum of <code>mask_replace_prob</code> and <code>random_replace_prob</code> must not exceed 1. If their sum is less than 1, the
remaining proportion will consist of masked tokens left unchanged.`;return{c(){h=l("p"),h.innerHTML=A,y=eo(`
<Example Options and Expectations>
`),w=l("ol"),w.innerHTML=j,W=o(),T=l("p"),T.innerHTML=ot},l(v){h=s(v,"P",{"data-svelte-h":!0}),d(h)!=="svelte-17ap8no"&&(h.innerHTML=A),y=to(v,`
<Example Options and Expectations>
`),w=s(v,"OL",{"data-svelte-h":!0}),d(w)!=="svelte-14xwpjc"&&(w.innerHTML=j),W=r(v),T=s(v,"P",{"data-svelte-h":!0}),d(T)!=="svelte-10bg7dp"&&(T.innerHTML=ot)},m(v,M){i(v,h,M),i(v,y,M),i(v,w,M),i(v,W,M),i(v,T,M)},p:Ga,d(v){v&&(a(h),a(y),a(w),a(W),a(T))}}}function ro(Me){let h,A=`This collator relies on details of the implementation of subword tokenization by <a href="/docs/transformers/pr_36839/ja/model_doc/bert#transformers.BertTokenizer">BertTokenizer</a>, specifically
that subword tokens are prefixed with <em>##</em>. For tokenizers that do not adhere to this scheme, this collator will
produce an output that is roughly equivalent to <code>.DataCollatorForLanguageModeling</code>.`;return{c(){h=l("p"),h.innerHTML=A},l(y){h=s(y,"P",{"data-svelte-h":!0}),d(h)!=="svelte-g7ut8w"&&(h.innerHTML=A)},m(y,w){i(y,h,w)},p:Ga,d(y){y&&a(h)}}}function no(Me){let h,A,y,w,j,W,T,ot=`データ照合器は、データセット要素のリストを入力として使用してバッチを形成するオブジェクトです。これらの要素は、
<code>train_dataset</code> または <code>eval_dataset</code> の要素と同じ型。`,v,M,ba=`バッチを構築できるようにするために、データ照合者は何らかの処理 (パディングなど) を適用する場合があります。そのうちのいくつかは（
<code>DataCollat​​orForLanguageModeling</code>) ランダムなデータ拡張 (ランダム マスキングなど) も適用します
形成されたバッチ上で。`,nt,Z,ka='使用例は、<a href="../examples">サンプル スクリプト</a> または <a href="../notebooks">サンプル ノートブック</a> にあります。',lt,ee,st,L,te,zt,Le,va=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`,At,Fe,$a="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",jt,qe,xa=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it’s useful.`,it,ae,dt,D,oe,It,Se,Ca=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`,Et,ze,ya="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",Ht,Ae,Ta=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it’s useful.`,Wt,je,Da=`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be
helpful if you need to set a return_tensors value at initialization.`,mt,re,pt,V,ne,Vt,Ie,wa="Data collator that will dynamically pad the inputs received.",ct,le,gt,N,se,Nt,Ee,Pa="Data collator that will dynamically pad the inputs received, as well as the labels.",ft,ie,ut,O,de,Ot,He,Ma="Data collator that will dynamically pad the inputs received, as well as the labels.",ht,me,_t,$,pe,Kt,We,La=`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`,Bt,B,Ut,U,ce,Rt,Ve,Fa="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",Gt,R,ge,Xt,Ne,qa="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",Jt,G,fe,Qt,Oe,Sa="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",bt,ue,kt,b,he,Yt,Ke,za="Data collator used for language modeling that masks entire words.",Zt,Be,Aa="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for masked language modeling</li>",ea,X,ta,J,_e,aa,Ue,ja=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,oa,Q,be,ra,Re,Ia=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,na,Y,ke,la,Ge,Ea=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,vt,ve,$t,x,$e,sa,Xe,Ha="Data collator used for permutation language modeling.",ia,Je,Wa="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for permutation language modeling with procedures specific to XLNet</li>",da,I,xe,ma,Qe,Va="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",pa,Ce,Na=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,ca,E,ye,ga,Ye,Oa="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",fa,Te,Ka=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,ua,H,De,ha,Ze,Ba="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",_a,we,Ua=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,xt,Pe,Ct,rt,yt;return j=new K({props:{title:"データ照合者",local:"データ照合者",headingTag:"h1"}}),ee=new K({props:{title:"Default data collator",local:"transformers.default_data_collator",headingTag:"h2"}}),te=new C({props:{name:"transformers.default_data_collator",anchor:"transformers.default_data_collator",parameters:[{name:"features",val:": typing.List[transformers.data.data_collator.InputDataClass]"},{name:"return_tensors",val:" = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L74"}}),ae=new K({props:{title:"DefaultDataCollator",local:"transformers.DefaultDataCollator",headingTag:"h2"}}),oe=new C({props:{name:"class transformers.DefaultDataCollator",anchor:"transformers.DefaultDataCollator",parameters:[{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DefaultDataCollator.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L99"}}),re=new K({props:{title:"DataCollatorWithPadding",local:"transformers.DataCollatorWithPadding",headingTag:"h2"}}),ne=new C({props:{name:"class transformers.DataCollatorWithPadding",anchor:"transformers.DataCollatorWithPadding",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorWithPadding.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorWithPadding.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_36839/ja/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorWithPadding.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorWithPadding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorWithPadding.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L236"}}),le=new K({props:{title:"DataCollatorForTokenClassification",local:"transformers.DataCollatorForTokenClassification",headingTag:"h2"}}),se=new C({props:{name:"class transformers.DataCollatorForTokenClassification",anchor:"transformers.DataCollatorForTokenClassification",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForTokenClassification.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForTokenClassification.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_36839/ja/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForTokenClassification.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForTokenClassification.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForTokenClassification.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForTokenClassification.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L288"}}),ie=new K({props:{title:"DataCollatorForSeq2Seq",local:"transformers.DataCollatorForSeq2Seq",headingTag:"h2"}}),de=new C({props:{name:"class transformers.DataCollatorForSeq2Seq",anchor:"transformers.DataCollatorForSeq2Seq",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"model",val:": typing.Optional[typing.Any] = None"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForSeq2Seq.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForSeq2Seq.model",description:`<strong>model</strong> (<a href="/docs/transformers/pr_36839/ja/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>, <em>optional</em>) &#x2014;
The model that is being trained. If set and has the <em>prepare_decoder_input_ids_from_labels</em>, use it to
prepare the <em>decoder_input_ids</em></p>
<p>This is useful when using <em>label_smoothing</em> to avoid calculating loss twice.`,name:"model"},{anchor:"transformers.DataCollatorForSeq2Seq.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/pr_36839/ja/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForSeq2Seq.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForSeq2Seq.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForSeq2Seq.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForSeq2Seq.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L626"}}),me=new K({props:{title:"DataCollatorForLanguageModeling",local:"transformers.DataCollatorForLanguageModeling",headingTag:"h2"}}),pe=new C({props:{name:"class transformers.DataCollatorForLanguageModeling",anchor:"transformers.DataCollatorForLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"mask_replace_prob",val:": float = 0.8"},{name:"random_replace_prob",val:": float = 0.1"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForLanguageModeling.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/pr_36839/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm",description:`<strong>mlm</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to use masked language modeling. If set to <code>False</code>, the labels are the same as the inputs
with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked
tokens and the value to predict for the masked token.`,name:"mlm"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm_probability",description:`<strong>mlm_probability</strong> (<code>float</code>, <em>optional</em>, defaults to 0.15) &#x2014;
The probability with which to (randomly) mask tokens in the input, when <code>mlm</code> is set to <code>True</code>.`,name:"mlm_probability"},{anchor:"transformers.DataCollatorForLanguageModeling.mask_replace_prob",description:`<strong>mask_replace_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.8) &#x2014;
The probability with which masked tokens are replaced by the tokenizer&#x2019;s mask token (e.g., <code>[MASK]</code>).
Defaults to 0.8, meaning 80% of the masked tokens will be replaced with <code>[MASK]</code>.
Only works when <code>mlm</code> is set to <code>True</code>.`,name:"mask_replace_prob"},{anchor:"transformers.DataCollatorForLanguageModeling.random_replace_prob",description:`<strong>random_replace_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The probability with which masked tokens are replaced by random tokens from the tokenizer&#x2019;s vocabulary.
Defaults to 0.1, meaning 10% of the masked tokens will be replaced with random tokens. The remaining
masked tokens (1 - mask_replace_prob - random_replace_prob) are left unchanged.
Only works when <code>mlm</code> is set to <code>True</code>.`,name:"random_replace_prob"},{anchor:"transformers.DataCollatorForLanguageModeling.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set, will pad the sequence to a multiple of the provided value.`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForLanguageModeling.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L762"}}),B=new Ra({props:{$$slots:{default:[oo]},$$scope:{ctx:Me}}}),ce=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1038"}}),ge=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"vocab_size",val:""},{name:"mask_token_id",val:""},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L862"}}),fe=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L968"}}),ue=new K({props:{title:"DataCollatorForWholeWordMask",local:"transformers.DataCollatorForWholeWordMask",headingTag:"h2"}}),he=new C({props:{name:"class transformers.DataCollatorForWholeWordMask",anchor:"transformers.DataCollatorForWholeWordMask",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"mask_replace_prob",val:": float = 0.8"},{name:"random_replace_prob",val:": float = 0.1"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1086"}}),X=new Ra({props:{$$slots:{default:[ro]},$$scope:{ctx:Me}}}),_e=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1317"}}),be=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1275"}}),ke=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1235"}}),ve=new K({props:{title:"DataCollatorForPermutationLanguageModeling",local:"transformers.DataCollatorForPermutationLanguageModeling",headingTag:"h2"}}),$e=new C({props:{name:"class transformers.DataCollatorForPermutationLanguageModeling",anchor:"transformers.DataCollatorForPermutationLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"plm_probability",val:": float = 0.16666666666666666"},{name:"max_span_length",val:": int = 5"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1449"}}),xe=new C({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1690"}}),ye=new C({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1583"}}),De=new C({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/vr_36839/src/transformers/data/data_collator.py#L1484"}}),Pe=new ao({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/ja/main_classes/data_collator.md"}}),{c(){h=l("meta"),A=o(),y=l("p"),w=o(),m(j.$$.fragment),W=o(),T=l("p"),T.innerHTML=ot,v=o(),M=l("p"),M.innerHTML=ba,nt=o(),Z=l("p"),Z.innerHTML=ka,lt=o(),m(ee.$$.fragment),st=o(),L=l("div"),m(te.$$.fragment),zt=o(),Le=l("p"),Le.textContent=va,At=o(),Fe=l("ul"),Fe.innerHTML=$a,jt=o(),qe=l("p"),qe.textContent=xa,it=o(),m(ae.$$.fragment),dt=o(),D=l("div"),m(oe.$$.fragment),It=o(),Se=l("p"),Se.textContent=Ca,Et=o(),ze=l("ul"),ze.innerHTML=ya,Ht=o(),Ae=l("p"),Ae.textContent=Ta,Wt=o(),je=l("p"),je.textContent=Da,mt=o(),m(re.$$.fragment),pt=o(),V=l("div"),m(ne.$$.fragment),Vt=o(),Ie=l("p"),Ie.textContent=wa,ct=o(),m(le.$$.fragment),gt=o(),N=l("div"),m(se.$$.fragment),Nt=o(),Ee=l("p"),Ee.textContent=Pa,ft=o(),m(ie.$$.fragment),ut=o(),O=l("div"),m(de.$$.fragment),Ot=o(),He=l("p"),He.textContent=Ma,ht=o(),m(me.$$.fragment),_t=o(),$=l("div"),m(pe.$$.fragment),Kt=o(),We=l("p"),We.textContent=La,Bt=o(),m(B.$$.fragment),Ut=o(),U=l("div"),m(ce.$$.fragment),Rt=o(),Ve=l("p"),Ve.textContent=Fa,Gt=o(),R=l("div"),m(ge.$$.fragment),Xt=o(),Ne=l("p"),Ne.textContent=qa,Jt=o(),G=l("div"),m(fe.$$.fragment),Qt=o(),Oe=l("p"),Oe.textContent=Sa,bt=o(),m(ue.$$.fragment),kt=o(),b=l("div"),m(he.$$.fragment),Yt=o(),Ke=l("p"),Ke.textContent=za,Zt=o(),Be=l("ul"),Be.innerHTML=Aa,ea=o(),m(X.$$.fragment),ta=o(),J=l("div"),m(_e.$$.fragment),aa=o(),Ue=l("p"),Ue.textContent=ja,oa=o(),Q=l("div"),m(be.$$.fragment),ra=o(),Re=l("p"),Re.textContent=Ia,na=o(),Y=l("div"),m(ke.$$.fragment),la=o(),Ge=l("p"),Ge.textContent=Ea,vt=o(),m(ve.$$.fragment),$t=o(),x=l("div"),m($e.$$.fragment),sa=o(),Xe=l("p"),Xe.textContent=Ha,ia=o(),Je=l("ul"),Je.innerHTML=Wa,da=o(),I=l("div"),m(xe.$$.fragment),ma=o(),Qe=l("p"),Qe.textContent=Va,pa=o(),Ce=l("ol"),Ce.innerHTML=Na,ca=o(),E=l("div"),m(ye.$$.fragment),ga=o(),Ye=l("p"),Ye.textContent=Oa,fa=o(),Te=l("ol"),Te.innerHTML=Ka,ua=o(),H=l("div"),m(De.$$.fragment),ha=o(),Ze=l("p"),Ze.textContent=Ba,_a=o(),we=l("ol"),we.innerHTML=Ua,xt=o(),m(Pe.$$.fragment),Ct=o(),rt=l("p"),this.h()},l(e){const n=Za("svelte-u9bgzb",document.head);h=s(n,"META",{name:!0,content:!0}),n.forEach(a),A=r(e),y=s(e,"P",{}),k(y).forEach(a),w=r(e),p(j.$$.fragment,e),W=r(e),T=s(e,"P",{"data-svelte-h":!0}),d(T)!=="svelte-wq0mnv"&&(T.innerHTML=ot),v=r(e),M=s(e,"P",{"data-svelte-h":!0}),d(M)!=="svelte-c5nfv2"&&(M.innerHTML=ba),nt=r(e),Z=s(e,"P",{"data-svelte-h":!0}),d(Z)!=="svelte-qqt368"&&(Z.innerHTML=ka),lt=r(e),p(ee.$$.fragment,e),st=r(e),L=s(e,"DIV",{class:!0});var z=k(L);p(te.$$.fragment,z),zt=r(z),Le=s(z,"P",{"data-svelte-h":!0}),d(Le)!=="svelte-1hmsgsg"&&(Le.textContent=va),At=r(z),Fe=s(z,"UL",{"data-svelte-h":!0}),d(Fe)!=="svelte-1pq8qks"&&(Fe.innerHTML=$a),jt=r(z),qe=s(z,"P",{"data-svelte-h":!0}),d(qe)!=="svelte-1vi1gug"&&(qe.textContent=xa),z.forEach(a),it=r(e),p(ae.$$.fragment,e),dt=r(e),D=s(e,"DIV",{class:!0});var F=k(D);p(oe.$$.fragment,F),It=r(F),Se=s(F,"P",{"data-svelte-h":!0}),d(Se)!=="svelte-1hmsgsg"&&(Se.textContent=Ca),Et=r(F),ze=s(F,"UL",{"data-svelte-h":!0}),d(ze)!=="svelte-1pq8qks"&&(ze.innerHTML=ya),Ht=r(F),Ae=s(F,"P",{"data-svelte-h":!0}),d(Ae)!=="svelte-1vi1gug"&&(Ae.textContent=Ta),Wt=r(F),je=s(F,"P",{"data-svelte-h":!0}),d(je)!=="svelte-vfvbwr"&&(je.textContent=Da),F.forEach(a),mt=r(e),p(re.$$.fragment,e),pt=r(e),V=s(e,"DIV",{class:!0});var Tt=k(V);p(ne.$$.fragment,Tt),Vt=r(Tt),Ie=s(Tt,"P",{"data-svelte-h":!0}),d(Ie)!=="svelte-1iebpai"&&(Ie.textContent=wa),Tt.forEach(a),ct=r(e),p(le.$$.fragment,e),gt=r(e),N=s(e,"DIV",{class:!0});var Dt=k(N);p(se.$$.fragment,Dt),Nt=r(Dt),Ee=s(Dt,"P",{"data-svelte-h":!0}),d(Ee)!=="svelte-4uvw1w"&&(Ee.textContent=Pa),Dt.forEach(a),ft=r(e),p(ie.$$.fragment,e),ut=r(e),O=s(e,"DIV",{class:!0});var wt=k(O);p(de.$$.fragment,wt),Ot=r(wt),He=s(wt,"P",{"data-svelte-h":!0}),d(He)!=="svelte-4uvw1w"&&(He.textContent=Ma),wt.forEach(a),ht=r(e),p(me.$$.fragment,e),_t=r(e),$=s(e,"DIV",{class:!0});var q=k($);p(pe.$$.fragment,q),Kt=r(q),We=s(q,"P",{"data-svelte-h":!0}),d(We)!=="svelte-10km8jk"&&(We.textContent=La),Bt=r(q),p(B.$$.fragment,q),Ut=r(q),U=s(q,"DIV",{class:!0});var Pt=k(U);p(ce.$$.fragment,Pt),Rt=r(Pt),Ve=s(Pt,"P",{"data-svelte-h":!0}),d(Ve)!=="svelte-iv4xqf"&&(Ve.textContent=Fa),Pt.forEach(a),Gt=r(q),R=s(q,"DIV",{class:!0});var Mt=k(R);p(ge.$$.fragment,Mt),Xt=r(Mt),Ne=s(Mt,"P",{"data-svelte-h":!0}),d(Ne)!=="svelte-iv4xqf"&&(Ne.textContent=qa),Mt.forEach(a),Jt=r(q),G=s(q,"DIV",{class:!0});var Lt=k(G);p(fe.$$.fragment,Lt),Qt=r(Lt),Oe=s(Lt,"P",{"data-svelte-h":!0}),d(Oe)!=="svelte-iv4xqf"&&(Oe.textContent=Sa),Lt.forEach(a),q.forEach(a),bt=r(e),p(ue.$$.fragment,e),kt=r(e),b=s(e,"DIV",{class:!0});var P=k(b);p(he.$$.fragment,P),Yt=r(P),Ke=s(P,"P",{"data-svelte-h":!0}),d(Ke)!=="svelte-sjqk55"&&(Ke.textContent=za),Zt=r(P),Be=s(P,"UL",{"data-svelte-h":!0}),d(Be)!=="svelte-d2ozxj"&&(Be.innerHTML=Aa),ea=r(P),p(X.$$.fragment,P),ta=r(P),J=s(P,"DIV",{class:!0});var Ft=k(J);p(_e.$$.fragment,Ft),aa=r(Ft),Ue=s(Ft,"P",{"data-svelte-h":!0}),d(Ue)!=="svelte-10msjh1"&&(Ue.textContent=ja),Ft.forEach(a),oa=r(P),Q=s(P,"DIV",{class:!0});var qt=k(Q);p(be.$$.fragment,qt),ra=r(qt),Re=s(qt,"P",{"data-svelte-h":!0}),d(Re)!=="svelte-10msjh1"&&(Re.textContent=Ia),qt.forEach(a),na=r(P),Y=s(P,"DIV",{class:!0});var St=k(Y);p(ke.$$.fragment,St),la=r(St),Ge=s(St,"P",{"data-svelte-h":!0}),d(Ge)!=="svelte-10msjh1"&&(Ge.textContent=Ea),St.forEach(a),P.forEach(a),vt=r(e),p(ve.$$.fragment,e),$t=r(e),x=s(e,"DIV",{class:!0});var S=k(x);p($e.$$.fragment,S),sa=r(S),Xe=s(S,"P",{"data-svelte-h":!0}),d(Xe)!=="svelte-12inif1"&&(Xe.textContent=Ha),ia=r(S),Je=s(S,"UL",{"data-svelte-h":!0}),d(Je)!=="svelte-1t4qh7k"&&(Je.innerHTML=Wa),da=r(S),I=s(S,"DIV",{class:!0});var et=k(I);p(xe.$$.fragment,et),ma=r(et),Qe=s(et,"P",{"data-svelte-h":!0}),d(Qe)!=="svelte-1u70phx"&&(Qe.textContent=Va),pa=r(et),Ce=s(et,"OL",{start:!0,"data-svelte-h":!0}),d(Ce)!=="svelte-1r6jrbi"&&(Ce.innerHTML=Na),et.forEach(a),ca=r(S),E=s(S,"DIV",{class:!0});var tt=k(E);p(ye.$$.fragment,tt),ga=r(tt),Ye=s(tt,"P",{"data-svelte-h":!0}),d(Ye)!=="svelte-1u70phx"&&(Ye.textContent=Oa),fa=r(tt),Te=s(tt,"OL",{start:!0,"data-svelte-h":!0}),d(Te)!=="svelte-1r6jrbi"&&(Te.innerHTML=Ka),tt.forEach(a),ua=r(S),H=s(S,"DIV",{class:!0});var at=k(H);p(De.$$.fragment,at),ha=r(at),Ze=s(at,"P",{"data-svelte-h":!0}),d(Ze)!=="svelte-1u70phx"&&(Ze.textContent=Ba),_a=r(at),we=s(at,"OL",{start:!0,"data-svelte-h":!0}),d(we)!=="svelte-1r6jrbi"&&(we.innerHTML=Ua),at.forEach(a),S.forEach(a),xt=r(e),p(Pe.$$.fragment,e),Ct=r(e),rt=s(e,"P",{}),k(rt).forEach(a),this.h()},h(){_(h,"name","hf:doc:metadata"),_(h,"content",lo),_(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Ce,"start","0"),_(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Te,"start","0"),_(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(we,"start","0"),_(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){t(document.head,h),i(e,A,n),i(e,y,n),i(e,w,n),c(j,e,n),i(e,W,n),i(e,T,n),i(e,v,n),i(e,M,n),i(e,nt,n),i(e,Z,n),i(e,lt,n),c(ee,e,n),i(e,st,n),i(e,L,n),c(te,L,null),t(L,zt),t(L,Le),t(L,At),t(L,Fe),t(L,jt),t(L,qe),i(e,it,n),c(ae,e,n),i(e,dt,n),i(e,D,n),c(oe,D,null),t(D,It),t(D,Se),t(D,Et),t(D,ze),t(D,Ht),t(D,Ae),t(D,Wt),t(D,je),i(e,mt,n),c(re,e,n),i(e,pt,n),i(e,V,n),c(ne,V,null),t(V,Vt),t(V,Ie),i(e,ct,n),c(le,e,n),i(e,gt,n),i(e,N,n),c(se,N,null),t(N,Nt),t(N,Ee),i(e,ft,n),c(ie,e,n),i(e,ut,n),i(e,O,n),c(de,O,null),t(O,Ot),t(O,He),i(e,ht,n),c(me,e,n),i(e,_t,n),i(e,$,n),c(pe,$,null),t($,Kt),t($,We),t($,Bt),c(B,$,null),t($,Ut),t($,U),c(ce,U,null),t(U,Rt),t(U,Ve),t($,Gt),t($,R),c(ge,R,null),t(R,Xt),t(R,Ne),t($,Jt),t($,G),c(fe,G,null),t(G,Qt),t(G,Oe),i(e,bt,n),c(ue,e,n),i(e,kt,n),i(e,b,n),c(he,b,null),t(b,Yt),t(b,Ke),t(b,Zt),t(b,Be),t(b,ea),c(X,b,null),t(b,ta),t(b,J),c(_e,J,null),t(J,aa),t(J,Ue),t(b,oa),t(b,Q),c(be,Q,null),t(Q,ra),t(Q,Re),t(b,na),t(b,Y),c(ke,Y,null),t(Y,la),t(Y,Ge),i(e,vt,n),c(ve,e,n),i(e,$t,n),i(e,x,n),c($e,x,null),t(x,sa),t(x,Xe),t(x,ia),t(x,Je),t(x,da),t(x,I),c(xe,I,null),t(I,ma),t(I,Qe),t(I,pa),t(I,Ce),t(x,ca),t(x,E),c(ye,E,null),t(E,ga),t(E,Ye),t(E,fa),t(E,Te),t(x,ua),t(x,H),c(De,H,null),t(H,ha),t(H,Ze),t(H,_a),t(H,we),i(e,xt,n),c(Pe,e,n),i(e,Ct,n),i(e,rt,n),yt=!0},p(e,[n]){const z={};n&2&&(z.$$scope={dirty:n,ctx:e}),B.$set(z);const F={};n&2&&(F.$$scope={dirty:n,ctx:e}),X.$set(F)},i(e){yt||(g(j.$$.fragment,e),g(ee.$$.fragment,e),g(te.$$.fragment,e),g(ae.$$.fragment,e),g(oe.$$.fragment,e),g(re.$$.fragment,e),g(ne.$$.fragment,e),g(le.$$.fragment,e),g(se.$$.fragment,e),g(ie.$$.fragment,e),g(de.$$.fragment,e),g(me.$$.fragment,e),g(pe.$$.fragment,e),g(B.$$.fragment,e),g(ce.$$.fragment,e),g(ge.$$.fragment,e),g(fe.$$.fragment,e),g(ue.$$.fragment,e),g(he.$$.fragment,e),g(X.$$.fragment,e),g(_e.$$.fragment,e),g(be.$$.fragment,e),g(ke.$$.fragment,e),g(ve.$$.fragment,e),g($e.$$.fragment,e),g(xe.$$.fragment,e),g(ye.$$.fragment,e),g(De.$$.fragment,e),g(Pe.$$.fragment,e),yt=!0)},o(e){f(j.$$.fragment,e),f(ee.$$.fragment,e),f(te.$$.fragment,e),f(ae.$$.fragment,e),f(oe.$$.fragment,e),f(re.$$.fragment,e),f(ne.$$.fragment,e),f(le.$$.fragment,e),f(se.$$.fragment,e),f(ie.$$.fragment,e),f(de.$$.fragment,e),f(me.$$.fragment,e),f(pe.$$.fragment,e),f(B.$$.fragment,e),f(ce.$$.fragment,e),f(ge.$$.fragment,e),f(fe.$$.fragment,e),f(ue.$$.fragment,e),f(he.$$.fragment,e),f(X.$$.fragment,e),f(_e.$$.fragment,e),f(be.$$.fragment,e),f(ke.$$.fragment,e),f(ve.$$.fragment,e),f($e.$$.fragment,e),f(xe.$$.fragment,e),f(ye.$$.fragment,e),f(De.$$.fragment,e),f(Pe.$$.fragment,e),yt=!1},d(e){e&&(a(A),a(y),a(w),a(W),a(T),a(v),a(M),a(nt),a(Z),a(lt),a(st),a(L),a(it),a(dt),a(D),a(mt),a(pt),a(V),a(ct),a(gt),a(N),a(ft),a(ut),a(O),a(ht),a(_t),a($),a(bt),a(kt),a(b),a(vt),a($t),a(x),a(xt),a(Ct),a(rt)),a(h),u(j,e),u(ee,e),u(te),u(ae,e),u(oe),u(re,e),u(ne),u(le,e),u(se),u(ie,e),u(de),u(me,e),u(pe),u(B),u(ce),u(ge),u(fe),u(ue,e),u(he),u(X),u(_e),u(be),u(ke),u(ve,e),u($e),u(xe),u(ye),u(De),u(Pe,e)}}}const lo='{"title":"データ照合者","local":"データ照合者","sections":[{"title":"Default data collator","local":"transformers.default_data_collator","sections":[],"depth":2},{"title":"DefaultDataCollator","local":"transformers.DefaultDataCollator","sections":[],"depth":2},{"title":"DataCollatorWithPadding","local":"transformers.DataCollatorWithPadding","sections":[],"depth":2},{"title":"DataCollatorForTokenClassification","local":"transformers.DataCollatorForTokenClassification","sections":[],"depth":2},{"title":"DataCollatorForSeq2Seq","local":"transformers.DataCollatorForSeq2Seq","sections":[],"depth":2},{"title":"DataCollatorForLanguageModeling","local":"transformers.DataCollatorForLanguageModeling","sections":[],"depth":2},{"title":"DataCollatorForWholeWordMask","local":"transformers.DataCollatorForWholeWordMask","sections":[],"depth":2},{"title":"DataCollatorForPermutationLanguageModeling","local":"transformers.DataCollatorForPermutationLanguageModeling","sections":[],"depth":2}],"depth":1}';function so(Me){return Ja(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class fo extends Qa{constructor(h){super(),Ya(this,h,so,no,Xa,{})}}export{fo as component};