Buckets:
| import"../chunks/DsnmJJEf.js";import{i as Ae,h as He,C as Pe,H as c,b as Le,a as d,D as l,E as Oe,s as De}from"../chunks/qxQfBDOH.js";import{p as Ke,o as $e,s as e,f as p,a as i,b as eo,d as s,c as g,r as n,n as f}from"../chunks/Dpn4EqEf.js";import{T as h}from"../chunks/BRV980rc.js";import{E as m}from"../chunks/BD6i_DTX.js";import{H as Ue}from"../chunks/DwWDXVGi.js";const oo='{"title":"ELECTRA","local":"electra","sections":[{"title":"Notes","local":"notes","sections":[],"depth":2},{"title":"ElectraConfig","local":"transformers.ElectraConfig","sections":[],"depth":2},{"title":"ElectraTokenizer","local":"transformers.BertTokenizer","sections":[],"depth":2},{"title":"Electra specific outputs","local":"transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput","sections":[],"depth":2},{"title":"ElectraModel","local":"transformers.ElectraModel","sections":[],"depth":2},{"title":"ElectraForPreTraining","local":"transformers.ElectraForPreTraining","sections":[],"depth":2},{"title":"ElectraForCausalLM","local":"transformers.ElectraForCausalLM","sections":[],"depth":2},{"title":"ElectraForMaskedLM","local":"transformers.ElectraForMaskedLM","sections":[],"depth":2},{"title":"ElectraForSequenceClassification","local":"transformers.ElectraForSequenceClassification","sections":[],"depth":2},{"title":"ElectraForMultipleChoice","local":"transformers.ElectraForMultipleChoice","sections":[],"depth":2},{"title":"ElectraForTokenClassification","local":"transformers.ElectraForTokenClassification","sections":[],"depth":2},{"title":"ElectraForQuestionAnswering","local":"transformers.ElectraForQuestionAnswering","sections":[],"depth":2}],"depth":1}';var to=g('<meta name="hf:doc:metadata"/>'),so=g("<!> <!>",1),Ee=g("<p>Examples:</p> <!>",1),u=g(`<p>Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code> instance afterwards instead of this since the former takes care of running the pre and post processing steps while | |
| the latter silently ignores them.</p>`),_=g("<p>Example:</p> <!>",1),no=g("<p>Example of single-label classification:</p> <!>",1),ao=g("<p>Example of multi-label classification:</p> <!>",1),ro=g(`<p></p> <p><em>This model was published in HF papers on 2020-03-23 and contributed to Hugging Face Transformers on 2020-11-16.</em></p> <!> <div style="float: right;"><div class="flex flex-wrap space-x-1"><img alt="SDPA" src="https://img.shields.io/badge/SDPA-DE3412?style=flat&logo=pytorch&logoColor=white"/></div></div> <!> <p><a href="https://huggingface.co/papers/2003.10555" rel="nofollow">ELECTRA</a> modifies the pretraining objective of traditional masked language models like BERT. Instead of just masking tokens and asking the model to predict them, ELECTRA trains two models, a generator and a discriminator. The generator replaces some tokens with plausible alternatives and the discriminator (the model you’ll actually use) learns to detect which tokens are original and which were replaced. This training approach is very efficient and scales to larger models while using considerably less compute.</p> <p>This approach is super efficient because ELECTRA learns from every single token in the input, not just the masked ones. That’s why even the small ELECTRA models can match or outperform much larger models while using way less computing resources.</p> <p>You can find all the original ELECTRA checkpoints under the <a href="https://huggingface.co/collections/google/electra-release-64ff6e8b18830fabea30a1ab" rel="nofollow">ELECTRA</a> release.</p> <blockquote class="tip"><p>Click on the right sidebar for more examples of how to use ELECTRA for different language tasks like sequence classification, token classification, and question answering.</p></blockquote> <p>The example below demonstrates how to classify text with <a href="/docs/transformers/pr_46007/en/main_classes/pipelines#transformers.Pipeline">Pipeline</a> or the <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoModel">AutoModel</a> class.</p> <!> <!> <ul><li><p>ELECTRA consists of two transformer models, a generator (G) and a discriminator (D). For most downstream tasks, use the discriminator model (as indicated by <code>*-discriminator</code> in the name) rather than the generator.</p></li> <li><p>ELECTRA comes in three sizes: small (14M parameters), base (110M parameters), and large (335M parameters).</p></li> <li><p>ELECTRA can use a smaller embedding size than the hidden size for efficiency. When <code>embedding_size</code> is smaller than <code>hidden_size</code> in the configuration, a projection layer connects them.</p></li> <li><p>When using batched inputs with padding, make sure to use attention masks to prevent the model from attending to padding tokens.</p> <!></li> <li><p>When using the discriminator for a downstream task, you can load it into any of the ELECTRA model classes (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForSequenceClassification">ElectraForSequenceClassification</a>, <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForTokenClassification">ElectraForTokenClassification</a>, etc.).</p></li></ul> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This is the configuration class to store the configuration of a ElectraModel. It is used to instantiate a Electra | |
| model according to the specified arguments, defining the model architecture. Instantiating a configuration with the | |
| defaults will yield a similar configuration to that of the <a href="https://huggingface.co/google/electra-small-discriminator" rel="nofollow">google/electra-small-discriminator</a></p> <p>Configuration objects inherit from <a href="/docs/transformers/pr_46007/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> and can be used to control the model outputs. Read the | |
| documentation from <a href="/docs/transformers/pr_46007/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> for more information.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Construct a BERT tokenizer (backed by HuggingFace’s tokenizers library). Based on WordPiece.</p> <p>This tokenizer inherits from <a href="/docs/transformers/pr_46007/en/main_classes/tokenizer#transformers.TokenizersBackend">TokenizersBackend</a> which contains most of the main methods. Users should refer to | |
| this superclass for more information regarding those methods.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Output type of <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForPreTraining">ElectraForPreTraining</a>.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The bare Electra Model outputting raw hidden-states without any specific head on top.</p> <p>This model inherits from <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the | |
| library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads | |
| etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass. | |
| Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage | |
| and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraModel">ElectraModel</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) — Sequence of hidden-states at the output of the last layer of the model.</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.</p></li> <li><p><strong>cross_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> and <code>config.add_cross_attention=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights of the decoder’s cross-attention layer, after the attention softmax, used to compute the | |
| weighted average in the cross-attention heads.</p></li></ul></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Electra model with a binary classification head on top as used during pretraining for identifying generated tokens.</p> <p>It is recommended to load the discriminator checkpoint into that model.</p> <p>This model inherits from <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the | |
| library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads | |
| etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass. | |
| Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage | |
| and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForPreTraining">ElectraForPreTraining</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>*optional*</code>, returned when <code>labels</code> is provided, <code>torch.FloatTensor</code> of shape <code>(1,)</code>) — Total loss of the ELECTRA objective.</p></li> <li><p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>) — Prediction scores of the head (scores for each token before SoftMax).</p></li> <li><p><strong>hidden_states</strong> (<code>tuple[torch.FloatTensor]</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple[torch.FloatTensor]</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.</p></li></ul> <!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>ELECTRA Model with a <code>language modeling</code> head on top for CLM fine-tuning.</p> <p>This model inherits from <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the | |
| library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads | |
| etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass. | |
| Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage | |
| and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForCausalLM">ElectraForCausalLM</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p></li> <li><p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.</p></li> <li><p><strong>cross_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Cross attentions weights after the attention softmax, used to compute the weighted average in the | |
| cross-attention heads.</p></li> <li><p><strong>past_key_values</strong> (<code>Cache</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — It is a <a href="/docs/transformers/pr_46007/en/internal/generation_utils#transformers.Cache">Cache</a> instance. For more details, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>.</p> <p>Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p></li></ul> <!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Electra model with a language modeling head on top.</p> <p>Even though both the discriminator and generator may be loaded into this model, the generator is the only model of | |
| the two to have been trained for the masked language modeling task.</p> <p>This model inherits from <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the | |
| library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads | |
| etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass. | |
| Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage | |
| and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForMaskedLM">ElectraForMaskedLM</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Masked language modeling (MLM) loss.</p></li> <li><p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.</p></li></ul> <!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the | |
| pooled output) e.g. for GLUE tasks.</p> <p>This model inherits from <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the | |
| library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads | |
| etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass. | |
| Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage | |
| and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForSequenceClassification">ElectraForSequenceClassification</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Classification (or regression if config.num_labels==1) loss.</p></li> <li><p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, config.num_labels)</code>) — Classification (or regression if config.num_labels==1) scores (before SoftMax).</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.</p></li></ul> <!> <!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The Electra Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a | |
| softmax) e.g. for RocStories/SWAG tasks.</p> <p>This model inherits from <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the | |
| library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads | |
| etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass. | |
| Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage | |
| and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForMultipleChoice">ElectraForMultipleChoice</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <em>(1,)</em>, <em>optional</em>, returned when <code>labels</code> is provided) — Classification loss.</p></li> <li><p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_choices)</code>) — <em>num_choices</em> is the second dimension of the input tensors. (see <em>input_ids</em> above).</p> <p>Classification scores (before SoftMax).</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.</p></li></ul> <!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Electra model with a token classification head on top.</p> <p>Both the discriminator and generator may be loaded into this model.</p> <p>This model inherits from <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the | |
| library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads | |
| etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass. | |
| Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage | |
| and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForTokenClassification">ElectraForTokenClassification</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Classification loss.</p></li> <li><p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.num_labels)</code>) — Classification scores (before SoftMax).</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.</p></li></ul> <!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The Electra transformer with a span classification head on top for extractive question-answering tasks like | |
| SQuAD (a linear layer on top of the hidden-states output to compute <code>span start logits</code> and <code>span end logits</code>).</p> <p>This model inherits from <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the | |
| library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads | |
| etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass. | |
| Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage | |
| and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForQuestionAnswering">ElectraForQuestionAnswering</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.</p></li> <li><p><strong>start_logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>) — Span-start scores (before SoftMax).</p></li> <li><p><strong>end_logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>) — Span-end scores (before SoftMax).</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.</p></li></ul> <!></div></div> <!> <p></p>`,1);function uo(ze,Ce){Ke(Ce,!1),$e(()=>{new URLSearchParams(window.location.search).get("fw")}),Ae();var F=ro();He("vejje6",t=>{var a=to();De(a,"content",oo),i(t,a)});var Z=e(p(F),4);Pe(Z,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var W=e(Z,4);c(W,{title:"ELECTRA",local:"electra",headingTag:"h1"});var I=e(W,12);Le(I,{id:"usage",options:["Pipeline","AutoModel"],children:(t,a)=>{var o=so(),r=p(o);Ue(r,{id:"usage",option:"Pipeline",children:(C,Ye)=>{d(C,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBJTBBY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKCUwQSUyMCUyMCUyMCUyMHRhc2slM0QlMjJ0ZXh0LWNsYXNzaWZpY2F0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJiaGFkcmVzaC1zYXZhbmklMkZlbGVjdHJhLWJhc2UtZW1vdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZSUzRDAlMEEpJTBBY2xhc3NpZmllciglMjJUaGlzJTIwcmVzdGF1cmFudCUyMGhhcyUyMGFtYXppbmclMjBmb29kISUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline | |
| classifier = pipeline( | |
| task=<span class="hljs-string">"text-classification"</span>, | |
| model=<span class="hljs-string">"bhadresh-savani/electra-base-emotion"</span>, | |
| device=<span class="hljs-number">0</span> | |
| ) | |
| classifier(<span class="hljs-string">"This restaurant has amazing food!"</span>)`,lang:"python",wrap:!1})},$$slots:{default:!0}});var Qe=e(r,2);Ue(Qe,{id:"usage",option:"AutoModel",children:(C,Ye)=>{d(C,{code:"aW1wb3J0JTIwdG9yY2glMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaGFkcmVzaC1zYXZhbmklMkZlbGVjdHJhLWJhc2UtZW1vdGlvbiUyMiUyQyUwQSklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJoYWRyZXNoLXNhdmFuaSUyRmVsZWN0cmEtYmFzZS1lbW90aW9uJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEEpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMkVMRUNUUkElMjBpcyUyMG1vcmUlMjBlZmZpY2llbnQlMjB0aGFuJTIwQkVSVCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKG1vZGVsLmRldmljZSklMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQSUyMCUyMCUyMCUyMGxvZ2l0cyUyMCUzRCUyMG91dHB1dHMubG9naXRzJTBBJTIwJTIwJTIwJTIwcHJlZGljdGVkX2NsYXNzX2lkJTIwJTNEJTIwbG9naXRzLmFyZ21heChkaW0lM0QtMSkuaXRlbSgpJTBBJTIwJTIwJTIwJTIwcHJlZGljdGVkX2xhYmVsJTIwJTNEJTIwbW9kZWwuY29uZmlnLmlkMmxhYmVsJTVCcHJlZGljdGVkX2NsYXNzX2lkJTVEJTBBcHJpbnQoZiUyMlByZWRpY3RlZCUyMGxhYmVsJTNBJTIwJTdCcHJlZGljdGVkX2xhYmVsJTdEJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification, AutoTokenizer | |
| tokenizer = AutoTokenizer.from_pretrained( | |
| <span class="hljs-string">"bhadresh-savani/electra-base-emotion"</span>, | |
| ) | |
| model = AutoModelForSequenceClassification.from_pretrained( | |
| <span class="hljs-string">"bhadresh-savani/electra-base-emotion"</span>, device_map=<span class="hljs-string">"auto"</span>, | |
| ) | |
| inputs = tokenizer(<span class="hljs-string">"ELECTRA is more efficient than BERT"</span>, return_tensors=<span class="hljs-string">"pt"</span>).to(model.device) | |
| <span class="hljs-keyword">with</span> torch.no_grad(): | |
| outputs = model(**inputs) | |
| logits = outputs.logits | |
| predicted_class_id = logits.argmax(dim=-<span class="hljs-number">1</span>).item() | |
| predicted_label = model.config.id2label[predicted_class_id] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Predicted label: <span class="hljs-subst">{predicted_label}</span>"</span>)`,lang:"python",wrap:!1})},$$slots:{default:!0}}),i(t,o)},$$slots:{default:!0}});var x=e(I,2);c(x,{title:"Notes",local:"notes",headingTag:"h2"});var b=e(x,2),G=e(s(b),6),Fe=e(s(G),2);d(Fe,{code:"JTIzJTIwRXhhbXBsZSUyMG9mJTIwcHJvcGVybHklMjBoYW5kbGluZyUyMHBhZGRpbmclMjB3aXRoJTIwYXR0ZW50aW9uJTIwbWFza3MlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTVCJTIyU2hvcnQlMjB0ZXh0JTIyJTJDJTIwJTIyVGhpcyUyMGlzJTIwYSUyMG11Y2glMjBsb25nZXIlMjB0ZXh0JTIwdGhhdCUyMG5lZWRzJTIwcGFkZGluZyUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhZGRpbmclM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpJTIwJTIwJTIzJTIwYXV0b21hdGljYWxseSUyMHVzZXMlMjB0aGUlMjBhdHRlbnRpb25fbWFzaw==",highlighted:`<span class="hljs-comment"># Example of properly handling padding with attention masks</span> | |
| inputs = tokenizer([<span class="hljs-string">"Short text"</span>, <span class="hljs-string">"This is a much longer text that needs padding"</span>], | |
| padding=<span class="hljs-literal">True</span>, | |
| return_tensors=<span class="hljs-string">"pt"</span>) | |
| outputs = model(**inputs) <span class="hljs-comment"># automatically uses the attention_mask</span>`,lang:"py",wrap:!1}),n(G),f(2),n(b);var B=e(b,2);c(B,{title:"ElectraConfig",local:"transformers.ElectraConfig",headingTag:"h2"});var y=e(B,2),q=s(y);l(q,{name:"class transformers.ElectraConfig",anchor:"transformers.ElectraConfig",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/configuration_electra.py#L25",parameters:[{name:"transformers_version",val:": str | None = None"},{name:"architectures",val:": list[str] | None = None"},{name:"output_hidden_states",val:": bool | None = False"},{name:"return_dict",val:": bool | None = True"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"chunk_size_feed_forward",val:": int = 0"},{name:"is_encoder_decoder",val:": bool = False"},{name:"id2label",val:": dict[int, str] | dict[str, str] | None = None"},{name:"label2id",val:": dict[str, int] | dict[str, str] | None = None"},{name:"problem_type",val:": Literal['regression', 'single_label_classification', 'multi_label_classification'] | None = None"},{name:"vocab_size",val:": int = 30522"},{name:"embedding_size",val:": int = 128"},{name:"hidden_size",val:": int = 256"},{name:"num_hidden_layers",val:": int = 12"},{name:"num_attention_heads",val:": int = 4"},{name:"intermediate_size",val:": int = 1024"},{name:"hidden_act",val:": str = 'gelu'"},{name:"hidden_dropout_prob",val:": float | int = 0.1"},{name:"attention_probs_dropout_prob",val:": float | int = 0.1"},{name:"max_position_embeddings",val:": int = 512"},{name:"type_vocab_size",val:": int = 2"},{name:"initializer_range",val:": float = 0.02"},{name:"layer_norm_eps",val:": float = 1e-12"},{name:"summary_type",val:": str = 'first'"},{name:"summary_use_proj",val:": bool = True"},{name:"summary_activation",val:": str = 'gelu'"},{name:"summary_last_dropout",val:": float | int = 0.1"},{name:"pad_token_id",val:": int | None = 0"},{name:"use_cache",val:": bool = True"},{name:"classifier_dropout",val:": float | int | None = None"},{name:"is_decoder",val:": bool = False"},{name:"add_cross_attention",val:": bool = False"},{name:"bos_token_id",val:": int | None = None"},{name:"eos_token_id",val:": int | list[int] | None = None"},{name:"tie_word_embeddings",val:": bool = True"}],parametersDescription:[{anchor:"transformers.ElectraConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>30522</code>) — | |
| Vocabulary size of the model. Defines the number of different tokens that can be represented by the <code>input_ids</code>.`,name:"vocab_size"},{anchor:"transformers.ElectraConfig.embedding_size",description:`<strong>embedding_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>128</code>) — | |
| Dimensionality of the embeddings and hidden states.`,name:"embedding_size"},{anchor:"transformers.ElectraConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>256</code>) — | |
| Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.ElectraConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to <code>12</code>) — | |
| Number of hidden layers in the Transformer decoder.`,name:"num_hidden_layers"},{anchor:"transformers.ElectraConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to <code>4</code>) — | |
| Number of attention heads for each attention layer in the Transformer decoder.`,name:"num_attention_heads"},{anchor:"transformers.ElectraConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1024</code>) — | |
| Dimension of the MLP representations.`,name:"intermediate_size"},{anchor:"transformers.ElectraConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>gelu</code>) — | |
| The non-linear activation function (function or string) in the decoder. For example, <code>"gelu"</code>, | |
| <code>"relu"</code>, <code>"silu"</code>, etc.`,name:"hidden_act"},{anchor:"transformers.ElectraConfig.hidden_dropout_prob",description:`<strong>hidden_dropout_prob</strong> (<code>Union[float, int]</code>, <em>optional</em>, defaults to <code>0.1</code>) — | |
| The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.`,name:"hidden_dropout_prob"},{anchor:"transformers.ElectraConfig.attention_probs_dropout_prob",description:`<strong>attention_probs_dropout_prob</strong> (<code>Union[float, int]</code>, <em>optional</em>, defaults to <code>0.1</code>) — | |
| The dropout ratio for the attention probabilities.`,name:"attention_probs_dropout_prob"},{anchor:"transformers.ElectraConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to <code>512</code>) — | |
| The maximum sequence length that this model might ever be used with.`,name:"max_position_embeddings"},{anchor:"transformers.ElectraConfig.type_vocab_size",description:`<strong>type_vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2</code>) — | |
| The vocabulary size of the <code>token_type_ids</code>.`,name:"type_vocab_size"},{anchor:"transformers.ElectraConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.02</code>) — | |
| The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.ElectraConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-12</code>) — | |
| The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.ElectraConfig.summary_type",description:`<strong>summary_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"first"</code>) — | |
| Argument used when doing sequence summary. Used in the sequence classification and multiple choice models. | |
| Has to be one of the following options: | |
| <ul> | |
| <li><code>"last"</code>: Take the last token hidden state (like XLNet).</li> | |
| <li><code>"first"</code>: Take the first token hidden state (like BERT).</li> | |
| <li><code>"mean"</code>: Take the mean of all tokens hidden states.</li> | |
| <li><code>"cls_index"</code>: Supply a Tensor of classification token position (like GPT/GPT-2).</li> | |
| <li><code>"attn"</code>: Not implemented now, use multi-head attention.</li> | |
| </ul>`,name:"summary_type"},{anchor:"transformers.ElectraConfig.summary_use_proj",description:`<strong>summary_use_proj</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Argument used when doing sequence summary. Used in the sequence classification and multiple choice models. | |
| Whether or not to add a projection after the vector extraction.`,name:"summary_use_proj"},{anchor:"transformers.ElectraConfig.summary_activation",description:`<strong>summary_activation</strong> (<code>str</code>, <em>optional</em>) — | |
| Argument used when doing sequence summary. Used in the sequence classification and multiple choice models. | |
| Pass <code>"gelu"</code> for a gelu activation to the output, any other value will result in no activation.`,name:"summary_activation"},{anchor:"transformers.ElectraConfig.summary_last_dropout",description:`<strong>summary_last_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) — | |
| Argument used when doing sequence summary. Used in the sequence classification and multiple choice models. | |
| The dropout ratio to be used after the projection and activation.`,name:"summary_last_dropout"},{anchor:"transformers.ElectraConfig.pad_token_id",description:`<strong>pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to <code>0</code>) — | |
| Token id used for padding in the vocabulary.`,name:"pad_token_id"},{anchor:"transformers.ElectraConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not the model should return the last key/values attentions (not used by all models). Only | |
| relevant if <code>config.is_decoder=True</code> or when the model is a decoder-only generative model.`,name:"use_cache"},{anchor:"transformers.ElectraConfig.classifier_dropout",description:`<strong>classifier_dropout</strong> (<code>Union[float, int]</code>, <em>optional</em>) — | |
| The dropout ratio for classifier.`,name:"classifier_dropout"},{anchor:"transformers.ElectraConfig.is_decoder",description:`<strong>is_decoder</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether the model is used as a decoder or not. If <code>False</code>, the model is used as an encoder.`,name:"is_decoder"},{anchor:"transformers.ElectraConfig.add_cross_attention",description:`<strong>add_cross_attention</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether cross-attention layers should be added to the model.`,name:"add_cross_attention"},{anchor:"transformers.ElectraConfig.bos_token_id",description:`<strong>bos_token_id</strong> (<code>int</code>, <em>optional</em>) — | |
| Token id used for beginning-of-stream in the vocabulary.`,name:"bos_token_id"},{anchor:"transformers.ElectraConfig.eos_token_id",description:`<strong>eos_token_id</strong> (<code>Union[int, list[int]]</code>, <em>optional</em>) — | |
| Token id used for end-of-stream in the vocabulary.`,name:"eos_token_id"},{anchor:"transformers.ElectraConfig.tie_word_embeddings",description:`<strong>tie_word_embeddings</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether to tie weight embeddings according to model’s <code>tied_weights_keys</code> mapping.`,name:"tie_word_embeddings"}]});var Ze=e(q,6);m(Ze,{anchor:"transformers.ElectraConfig.example",children:(t,a)=>{var o=Ee(),r=e(p(o),2);d(r,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEVsZWN0cmFDb25maWclMkMlMjBFbGVjdHJhTW9kZWwlMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwRUxFQ1RSQSUyMGVsZWN0cmEtYmFzZS11bmNhc2VkJTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMEVsZWN0cmFDb25maWcoKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBtb2RlbCUyMCh3aXRoJTIwcmFuZG9tJTIwd2VpZ2h0cyklMjBmcm9tJTIwdGhlJTIwZWxlY3RyYS1iYXNlLXVuY2FzZWQlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEVsZWN0cmFNb2RlbChjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> ElectraConfig, ElectraModel | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Initializing a ELECTRA electra-base-uncased style configuration</span> | |
| <span class="hljs-meta">>>> </span>configuration = ElectraConfig() | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Initializing a model (with random weights) from the electra-base-uncased style configuration</span> | |
| <span class="hljs-meta">>>> </span>model = ElectraModel(configuration) | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Accessing the model configuration</span> | |
| <span class="hljs-meta">>>> </span>configuration = model.config`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}}),n(y);var R=e(y,2);c(R,{title:"ElectraTokenizer",local:"transformers.BertTokenizer",headingTag:"h2"});var T=e(R,2),We=s(T);l(We,{name:"class transformers.BertTokenizer",anchor:"transformers.BertTokenizer",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/bert/tokenization_bert.py#L41",parameters:[{name:"vocab",val:": str | dict[str, int] | None = None"},{name:"do_lower_case",val:": bool = True"},{name:"unk_token",val:": str = '[UNK]'"},{name:"sep_token",val:": str = '[SEP]'"},{name:"pad_token",val:": str = '[PAD]'"},{name:"cls_token",val:": str = '[CLS]'"},{name:"mask_token",val:": str = '[MASK]'"},{name:"tokenize_chinese_chars",val:": bool = True"},{name:"strip_accents",val:": bool | None = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BertTokenizer.vocab",description:`<strong>vocab</strong> (<code>str</code> or <code>dict[str, int]</code>, <em>optional</em>) — | |
| Custom vocabulary dictionary. If not provided, vocabulary is loaded from <code>vocab_file</code>.`,name:"vocab"},{anchor:"transformers.BertTokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to lowercase the input when tokenizing.`,name:"do_lower_case"},{anchor:"transformers.BertTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"[UNK]"</code>) — | |
| The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this | |
| token instead.`,name:"unk_token"},{anchor:"transformers.BertTokenizer.sep_token",description:`<strong>sep_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"[SEP]"</code>) — | |
| The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for | |
| sequence classification or for a text and a question for question answering. It is also used as the last | |
| token of a sequence built with special tokens.`,name:"sep_token"},{anchor:"transformers.BertTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"[PAD]"</code>) — | |
| The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.BertTokenizer.cls_token",description:`<strong>cls_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"[CLS]"</code>) — | |
| The classifier token which is used when doing sequence classification (classification of the whole sequence | |
| instead of per-token classification). It is the first token of the sequence when built with special tokens.`,name:"cls_token"},{anchor:"transformers.BertTokenizer.mask_token",description:`<strong>mask_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"[MASK]"</code>) — | |
| The token used for masking values. This is the token used when training this model with masked language | |
| modeling. This is the token which the model will try to predict.`,name:"mask_token"},{anchor:"transformers.BertTokenizer.tokenize_chinese_chars",description:`<strong>tokenize_chinese_chars</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to tokenize Chinese characters.`,name:"tokenize_chinese_chars"},{anchor:"transformers.BertTokenizer.strip_accents",description:`<strong>strip_accents</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to strip all accents. If this option is not specified, then it will be determined by the | |
| value for <code>lowercase</code> (as in the original BERT).`,name:"strip_accents"}]}),f(4),n(T);var N=e(T,2);c(N,{title:"Electra specific outputs",local:"transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput",headingTag:"h2"});var M=e(N,2),Ie=s(M);l(Ie,{name:"class transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput",anchor:"transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L530",parameters:[{name:"loss",val:": typing.Optional[torch.FloatTensor] = None"},{name:"logits",val:": typing.Optional[torch.FloatTensor] = None"},{name:"hidden_states",val:": tuple[torch.FloatTensor] | None = None"},{name:"attentions",val:": tuple[torch.FloatTensor] | None = None"}],parametersDescription:[{anchor:"transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput.loss",description:`<strong>loss</strong> (<code>*optional*</code>, returned when <code>labels</code> is provided, <code>torch.FloatTensor</code> of shape <code>(1,)</code>) — | |
| Total loss of the ELECTRA objective.`,name:"loss"},{anchor:"transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput.logits",description:`<strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>) — | |
| Prediction scores of the head (scores for each token before SoftMax).`,name:"logits"},{anchor:"transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput.hidden_states",description:`<strong>hidden_states</strong> (<code>tuple[torch.FloatTensor]</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — | |
| Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, + | |
| one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> | |
| <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.`,name:"hidden_states"},{anchor:"transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput.attentions",description:`<strong>attentions</strong> (<code>tuple[torch.FloatTensor]</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — | |
| Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> | |
| <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention | |
| heads.`,name:"attentions"}]}),f(2),n(M);var V=e(M,2);c(V,{title:"ElectraModel",local:"transformers.ElectraModel",headingTag:"h2"});var k=e(V,2),X=s(k);l(X,{name:"class transformers.ElectraModel",anchor:"transformers.ElectraModel",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L545",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.ElectraModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraModel">ElectraModel</a>) — | |
| Model configuration class with all the parameters of the model. Initializing with a config file does not | |
| load the weights associated with the model, only the configuration. Check out the | |
| <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var S=e(X,8),Q=s(S);l(Q,{name:"forward",anchor:"transformers.ElectraModel.forward",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L565",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"encoder_hidden_states",val:": typing.Optional[torch.Tensor] = None"},{name:"encoder_attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"past_key_values",val:": list[torch.FloatTensor] | None = None"},{name:"use_cache",val:": bool | None = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.ElectraModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p> | |
| <p>Indices can be obtained using <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and | |
| <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.ElectraModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.ElectraModel.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 corresponds to a <em>sentence A</em> token,</li> | |
| <li>1 corresponds to a <em>sentence B</em> token.</li> | |
| </ul> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"token_type_ids"},{anchor:"transformers.ElectraModel.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p> | |
| <p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.ElectraModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This | |
| is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the | |
| model’s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.ElectraModel.forward.encoder_hidden_states",description:`<strong>encoder_hidden_states</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention | |
| if the model is configured as a decoder.`,name:"encoder_hidden_states"},{anchor:"transformers.ElectraModel.forward.encoder_attention_mask",description:`<strong>encoder_attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in | |
| the cross-attention if the model is configured as a decoder. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul>`,name:"encoder_attention_mask"},{anchor:"transformers.ElectraModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>list[torch.FloatTensor]</code>, <em>optional</em>) — | |
| Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention | |
| blocks) that can be used to speed up sequential decoding. This typically consists in the <code>past_key_values</code> | |
| returned by the model at a previous stage of decoding, when <code>use_cache=True</code> or <code>config.use_cache=True</code>.</p> | |
| <p>Only <a href="/docs/transformers/pr_46007/en/internal/generation_utils#transformers.Cache">Cache</a> instance is allowed as input, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>. | |
| If no <code>past_key_values</code> are passed, <a href="/docs/transformers/pr_46007/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> will be initialized by default.</p> | |
| <p>The model will output the same cache format that is fed as input.</p> | |
| <p>If <code>past_key_values</code> are used, the user is expected to input only unprocessed <code>input_ids</code> (those that don’t | |
| have their past key value states given to this model) of shape <code>(batch_size, unprocessed_length)</code> instead of all <code>input_ids</code> | |
| of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.ElectraModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) — | |
| If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see | |
| <code>past_key_values</code>).`,name:"use_cache"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithCrossAttentions" | |
| >BaseModelOutputWithCrossAttentions</a> or a tuple of | |
| <code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various | |
| elements depending on the configuration (<a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraConfig" | |
| >ElectraConfig</a>) and inputs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithCrossAttentions" | |
| >BaseModelOutputWithCrossAttentions</a> or <code>tuple(torch.FloatTensor)</code></p> | |
| `});var xe=e(Q,4);h(xe,{children:(t,a)=>{var o=u();i(t,o)},$$slots:{default:!0}}),f(2),n(S),n(k);var Y=e(k,2);c(Y,{title:"ElectraForPreTraining",local:"transformers.ElectraForPreTraining",headingTag:"h2"});var w=e(Y,2),A=s(w);l(A,{name:"class transformers.ElectraForPreTraining",anchor:"transformers.ElectraForPreTraining",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L869",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.ElectraForPreTraining.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForPreTraining">ElectraForPreTraining</a>) — | |
| Model configuration class with all the parameters of the model. Initializing with a config file does not | |
| load the weights associated with the model, only the configuration. Check out the | |
| <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var H=e(A,10),P=s(H);l(P,{name:"forward",anchor:"transformers.ElectraForPreTraining.forward",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L878",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"labels",val:": typing.Optional[torch.Tensor] = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.ElectraForPreTraining.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p> | |
| <p>Indices can be obtained using <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and | |
| <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.ElectraForPreTraining.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.ElectraForPreTraining.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 corresponds to a <em>sentence A</em> token,</li> | |
| <li>1 corresponds to a <em>sentence B</em> token.</li> | |
| </ul> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"token_type_ids"},{anchor:"transformers.ElectraForPreTraining.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p> | |
| <p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.ElectraForPreTraining.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This | |
| is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the | |
| model’s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.ElectraForPreTraining.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see <code>input_ids</code> docstring) | |
| Indices should be in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 indicates the token is an original token,</li> | |
| <li>1 indicates the token was replaced.</li> | |
| </ul>`,name:"labels"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput" | |
| >ElectraForPreTrainingOutput</a> or a tuple of | |
| <code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various | |
| elements depending on the configuration (<a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraConfig" | |
| >ElectraConfig</a>) and inputs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput" | |
| >ElectraForPreTrainingOutput</a> or <code>tuple(torch.FloatTensor)</code></p> | |
| `});var L=e(P,4);h(L,{children:(t,a)=>{var o=u();i(t,o)},$$slots:{default:!0}});var Ge=e(L,4);m(Ge,{anchor:"transformers.ElectraForPreTraining.forward.example",children:(t,a)=>{var o=Ee(),r=e(p(o),2);d(r,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEVsZWN0cmFGb3JQcmVUcmFpbmluZyUyQyUyMEF1dG9Ub2tlbml6ZXIlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQWRpc2NyaW1pbmF0b3IlMjAlM0QlMjBFbGVjdHJhRm9yUHJlVHJhaW5pbmcuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtYmFzZS1kaXNjcmltaW5hdG9yJTIyKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtYmFzZS1kaXNjcmltaW5hdG9yJTIyKSUwQSUwQXNlbnRlbmNlJTIwJTNEJTIwJTIyVGhlJTIwcXVpY2slMjBicm93biUyMGZveCUyMGp1bXBzJTIwb3ZlciUyMHRoZSUyMGxhenklMjBkb2clMjIlMEFmYWtlX3NlbnRlbmNlJTIwJTNEJTIwJTIyVGhlJTIwcXVpY2slMjBicm93biUyMGZveCUyMGZha2UlMjBvdmVyJTIwdGhlJTIwbGF6eSUyMGRvZyUyMiUwQSUwQWZha2VfdG9rZW5zJTIwJTNEJTIwdG9rZW5pemVyLnRva2VuaXplKGZha2Vfc2VudGVuY2UlMkMlMjBhZGRfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQWZha2VfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyLmVuY29kZShmYWtlX3NlbnRlbmNlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFkaXNjcmltaW5hdG9yX291dHB1dHMlMjAlM0QlMjBkaXNjcmltaW5hdG9yKGZha2VfaW5wdXRzKSUwQXByZWRpY3Rpb25zJTIwJTNEJTIwdG9yY2gucm91bmQoKHRvcmNoLnNpZ24oZGlzY3JpbWluYXRvcl9vdXRwdXRzJTVCMCU1RCklMjAlMkIlMjAxKSUyMCUyRiUyMDIpJTBBJTBBZmFrZV90b2tlbnMlMEElMEFwcmVkaWN0aW9ucy5zcXVlZXplKCkudG9saXN0KCk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> ElectraForPreTraining, AutoTokenizer | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span>discriminator = ElectraForPreTraining.from_pretrained(<span class="hljs-string">"google/electra-base-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google/electra-base-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>sentence = <span class="hljs-string">"The quick brown fox jumps over the lazy dog"</span> | |
| <span class="hljs-meta">>>> </span>fake_sentence = <span class="hljs-string">"The quick brown fox fake over the lazy dog"</span> | |
| <span class="hljs-meta">>>> </span>fake_tokens = tokenizer.tokenize(fake_sentence, add_special_tokens=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>fake_inputs = tokenizer.encode(fake_sentence, return_tensors=<span class="hljs-string">"pt"</span>) | |
| <span class="hljs-meta">>>> </span>discriminator_outputs = discriminator(fake_inputs) | |
| <span class="hljs-meta">>>> </span>predictions = torch.<span class="hljs-built_in">round</span>((torch.sign(discriminator_outputs[<span class="hljs-number">0</span>]) + <span class="hljs-number">1</span>) / <span class="hljs-number">2</span>) | |
| <span class="hljs-meta">>>> </span>fake_tokens | |
| [<span class="hljs-string">'[CLS]'</span>, <span class="hljs-string">'the'</span>, <span class="hljs-string">'quick'</span>, <span class="hljs-string">'brown'</span>, <span class="hljs-string">'fox'</span>, <span class="hljs-string">'fake'</span>, <span class="hljs-string">'over'</span>, <span class="hljs-string">'the'</span>, <span class="hljs-string">'lazy'</span>, <span class="hljs-string">'dog'</span>, <span class="hljs-string">'[SEP]'</span>] | |
| <span class="hljs-meta">>>> </span>predictions.squeeze().tolist() | |
| [<span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">1.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>]`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}}),n(H),n(w);var O=e(w,2);c(O,{title:"ElectraForCausalLM",local:"transformers.ElectraForCausalLM",headingTag:"h2"});var v=e(O,2),D=s(v);l(D,{name:"class transformers.ElectraForCausalLM",anchor:"transformers.ElectraForCausalLM",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L1247",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.ElectraForCausalLM.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForCausalLM">ElectraForCausalLM</a>) — | |
| Model configuration class with all the parameters of the model. Initializing with a config file does not | |
| load the weights associated with the model, only the configuration. Check out the | |
| <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var K=e(D,8),$=s(K);l($,{name:"forward",anchor:"transformers.ElectraForCausalLM.forward",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L1268",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"encoder_hidden_states",val:": typing.Optional[torch.Tensor] = None"},{name:"encoder_attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"labels",val:": typing.Optional[torch.Tensor] = None"},{name:"past_key_values",val:": transformers.cache_utils.Cache | None = None"},{name:"use_cache",val:": bool | None = None"},{name:"logits_to_keep",val:": typing.Union[int, torch.Tensor] = 0"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.ElectraForCausalLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p> | |
| <p>Indices can be obtained using <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and | |
| <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.ElectraForCausalLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.ElectraForCausalLM.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 corresponds to a <em>sentence A</em> token,</li> | |
| <li>1 corresponds to a <em>sentence B</em> token.</li> | |
| </ul> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"token_type_ids"},{anchor:"transformers.ElectraForCausalLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p> | |
| <p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.ElectraForCausalLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This | |
| is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the | |
| model’s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.ElectraForCausalLM.forward.encoder_hidden_states",description:`<strong>encoder_hidden_states</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention | |
| if the model is configured as a decoder.`,name:"encoder_hidden_states"},{anchor:"transformers.ElectraForCausalLM.forward.encoder_attention_mask",description:`<strong>encoder_attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in | |
| the cross-attention if the model is configured as a decoder. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul>`,name:"encoder_attention_mask"},{anchor:"transformers.ElectraForCausalLM.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in | |
| <code>[-100, 0, ..., config.vocab_size]</code> (see <code>input_ids</code> docstring) Tokens with indices set to <code>-100</code> are | |
| ignored (masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>`,name:"labels"},{anchor:"transformers.ElectraForCausalLM.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>~cache_utils.Cache</code>, <em>optional</em>) — | |
| Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention | |
| blocks) that can be used to speed up sequential decoding. This typically consists in the <code>past_key_values</code> | |
| returned by the model at a previous stage of decoding, when <code>use_cache=True</code> or <code>config.use_cache=True</code>.</p> | |
| <p>Only <a href="/docs/transformers/pr_46007/en/internal/generation_utils#transformers.Cache">Cache</a> instance is allowed as input, see our <a href="https://huggingface.co/docs/transformers/en/kv_cache" rel="nofollow">kv cache guide</a>. | |
| If no <code>past_key_values</code> are passed, <a href="/docs/transformers/pr_46007/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> will be initialized by default.</p> | |
| <p>The model will output the same cache format that is fed as input.</p> | |
| <p>If <code>past_key_values</code> are used, the user is expected to input only unprocessed <code>input_ids</code> (those that don’t | |
| have their past key value states given to this model) of shape <code>(batch_size, unprocessed_length)</code> instead of all <code>input_ids</code> | |
| of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.ElectraForCausalLM.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) — | |
| If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see | |
| <code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.ElectraForCausalLM.forward.logits_to_keep",description:`<strong>logits_to_keep</strong> (<code>Union[int, torch.Tensor]</code>, <em>optional</em>, defaults to <code>0</code>) — | |
| If an <code>int</code>, compute logits for the last <code>logits_to_keep</code> tokens. If <code>0</code>, calculate logits for all | |
| <code>input_ids</code> (special case). Only last token logits are needed for generation, and calculating them only for that | |
| token can save memory, which becomes pretty significant for long sequences or large vocabulary size. | |
| If a <code>torch.Tensor</code>, must be 1D corresponding to the indices to keep in the sequence length dimension. | |
| This is useful when using packed tensor format (single dimension for batch and sequence length).`,name:"logits_to_keep"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithCrossAttentions" | |
| >CausalLMOutputWithCrossAttentions</a> or a tuple of | |
| <code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various | |
| elements depending on the configuration (<a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraConfig" | |
| >ElectraConfig</a>) and inputs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithCrossAttentions" | |
| >CausalLMOutputWithCrossAttentions</a> or <code>tuple(torch.FloatTensor)</code></p> | |
| `});var ee=e($,4);h(ee,{children:(t,a)=>{var o=u();i(t,o)},$$slots:{default:!0}});var Be=e(ee,4);m(Be,{anchor:"transformers.ElectraForCausalLM.forward.example",children:(t,a)=>{var o=_(),r=e(p(o),2);d(r,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBFbGVjdHJhRm9yQ2F1c2FsTE0lMkMlMjBFbGVjdHJhQ29uZmlnJTBBaW1wb3J0JTIwdG9yY2glMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZlbGVjdHJhLWJhc2UtZ2VuZXJhdG9yJTIyKSUwQWNvbmZpZyUyMCUzRCUyMEVsZWN0cmFDb25maWcuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtYmFzZS1nZW5lcmF0b3IlMjIpJTBBY29uZmlnLmlzX2RlY29kZXIlMjAlM0QlMjBUcnVlJTBBbW9kZWwlMjAlM0QlMjBFbGVjdHJhRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtYmFzZS1nZW5lcmF0b3IlMjIlMkMlMjBjb25maWclM0Rjb25maWcpJTBBJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMkhlbGxvJTJDJTIwbXklMjBkb2clMjBpcyUyMGN1dGUlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEElMEFwcmVkaWN0aW9uX2xvZ2l0cyUyMCUzRCUyMG91dHB1dHMubG9naXRz",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, ElectraForCausalLM, ElectraConfig | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google/electra-base-generator"</span>) | |
| <span class="hljs-meta">>>> </span>config = ElectraConfig.from_pretrained(<span class="hljs-string">"google/electra-base-generator"</span>) | |
| <span class="hljs-meta">>>> </span>config.is_decoder = <span class="hljs-literal">True</span> | |
| <span class="hljs-meta">>>> </span>model = ElectraForCausalLM.from_pretrained(<span class="hljs-string">"google/electra-base-generator"</span>, config=config) | |
| <span class="hljs-meta">>>> </span>inputs = tokenizer(<span class="hljs-string">"Hello, my dog is cute"</span>, return_tensors=<span class="hljs-string">"pt"</span>) | |
| <span class="hljs-meta">>>> </span>outputs = model(**inputs) | |
| <span class="hljs-meta">>>> </span>prediction_logits = outputs.logits`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}}),n(K),n(v);var oe=e(v,2);c(oe,{title:"ElectraForMaskedLM",local:"transformers.ElectraForMaskedLM",headingTag:"h2"});var j=e(oe,2),te=s(j);l(te,{name:"class transformers.ElectraForMaskedLM",anchor:"transformers.ElectraForMaskedLM",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L961",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.ElectraForMaskedLM.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForMaskedLM">ElectraForMaskedLM</a>) — | |
| Model configuration class with all the parameters of the model. Initializing with a config file does not | |
| load the weights associated with the model, only the configuration. Check out the | |
| <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var se=e(te,10),ne=s(se);l(ne,{name:"forward",anchor:"transformers.ElectraForMaskedLM.forward",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L980",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"labels",val:": typing.Optional[torch.Tensor] = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.ElectraForMaskedLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p> | |
| <p>Indices can be obtained using <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and | |
| <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.ElectraForMaskedLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.ElectraForMaskedLM.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 corresponds to a <em>sentence A</em> token,</li> | |
| <li>1 corresponds to a <em>sentence B</em> token.</li> | |
| </ul> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"token_type_ids"},{anchor:"transformers.ElectraForMaskedLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p> | |
| <p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.ElectraForMaskedLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This | |
| is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the | |
| model’s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.ElectraForMaskedLM.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Labels for computing the masked language modeling loss. Indices should be in <code>[-100, 0, ..., config.vocab_size]</code> (see <code>input_ids</code> docstring) Tokens with indices set to <code>-100</code> are ignored (masked), the | |
| loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.`,name:"labels"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.MaskedLMOutput" | |
| >MaskedLMOutput</a> or a tuple of | |
| <code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various | |
| elements depending on the configuration (<a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraConfig" | |
| >ElectraConfig</a>) and inputs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.MaskedLMOutput" | |
| >MaskedLMOutput</a> or <code>tuple(torch.FloatTensor)</code></p> | |
| `});var ae=e(ne,4);h(ae,{children:(t,a)=>{var o=u();i(t,o)},$$slots:{default:!0}});var qe=e(ae,4);m(qe,{anchor:"transformers.ElectraForMaskedLM.forward.example",children:(t,a)=>{var o=_(),r=e(p(o),2);d(r,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBFbGVjdHJhRm9yTWFza2VkTE0lMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtc21hbGwtZGlzY3JpbWluYXRvciUyMiklMEFtb2RlbCUyMCUzRCUyMEVsZWN0cmFGb3JNYXNrZWRMTS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZWxlY3RyYS1zbWFsbC1kaXNjcmltaW5hdG9yJTIyKSUwQSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJUaGUlMjBjYXBpdGFsJTIwb2YlMjBGcmFuY2UlMjBpcyUyMCUzQ21hc2slM0UuJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwbG9naXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpLmxvZ2l0cyUwQSUwQSUyMyUyMHJldHJpZXZlJTIwaW5kZXglMjBvZiUyMCUzQ21hc2slM0UlMEFtYXNrX3Rva2VuX2luZGV4JTIwJTNEJTIwKGlucHV0cy5pbnB1dF9pZHMlMjAlM0QlM0QlMjB0b2tlbml6ZXIubWFza190b2tlbl9pZCklNUIwJTVELm5vbnplcm8oYXNfdHVwbGUlM0RUcnVlKSU1QjAlNUQlMEElMEFwcmVkaWN0ZWRfdG9rZW5faWQlMjAlM0QlMjBsb2dpdHMlNUIwJTJDJTIwbWFza190b2tlbl9pbmRleCU1RC5hcmdtYXgoYXhpcyUzRC0xKSUwQXRva2VuaXplci5kZWNvZGUocHJlZGljdGVkX3Rva2VuX2lkKSUwQSUwQWxhYmVscyUyMCUzRCUyMHRva2VuaXplciglMjJUaGUlMjBjYXBpdGFsJTIwb2YlMjBGcmFuY2UlMjBpcyUyMFBhcmlzLiUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTBBJTIzJTIwbWFzayUyMGxhYmVscyUyMG9mJTIwbm9uLSUzQ21hc2slM0UlMjB0b2tlbnMlMEFsYWJlbHMlMjAlM0QlMjB0b3JjaC53aGVyZShpbnB1dHMuaW5wdXRfaWRzJTIwJTNEJTNEJTIwdG9rZW5pemVyLm1hc2tfdG9rZW5faWQlMkMlMjBsYWJlbHMlMkMlMjAtMTAwKSUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyUyQyUyMGxhYmVscyUzRGxhYmVscyklMEFyb3VuZChvdXRwdXRzLmxvc3MuaXRlbSgpJTJDJTIwMik=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, ElectraForMaskedLM | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>model = ElectraForMaskedLM.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>inputs = tokenizer(<span class="hljs-string">"The capital of France is <mask>."</span>, return_tensors=<span class="hljs-string">"pt"</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad(): | |
| <span class="hljs-meta">... </span> logits = model(**inputs).logits | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># retrieve index of <mask></span> | |
| <span class="hljs-meta">>>> </span>mask_token_index = (inputs.input_ids == tokenizer.mask_token_id)[<span class="hljs-number">0</span>].nonzero(as_tuple=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>] | |
| <span class="hljs-meta">>>> </span>predicted_token_id = logits[<span class="hljs-number">0</span>, mask_token_index].argmax(axis=-<span class="hljs-number">1</span>) | |
| <span class="hljs-meta">>>> </span>tokenizer.decode(predicted_token_id) | |
| ... | |
| <span class="hljs-meta">>>> </span>labels = tokenizer(<span class="hljs-string">"The capital of France is Paris."</span>, return_tensors=<span class="hljs-string">"pt"</span>)[<span class="hljs-string">"input_ids"</span>] | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># mask labels of non-<mask> tokens</span> | |
| <span class="hljs-meta">>>> </span>labels = torch.where(inputs.input_ids == tokenizer.mask_token_id, labels, -<span class="hljs-number">100</span>) | |
| <span class="hljs-meta">>>> </span>outputs = model(**inputs, labels=labels) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">round</span>(outputs.loss.item(), <span class="hljs-number">2</span>) | |
| ...`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}}),n(se),n(j);var re=e(j,2);c(re,{title:"ElectraForSequenceClassification",local:"transformers.ElectraForSequenceClassification",headingTag:"h2"});var J=e(re,2),ie=s(J);l(ie,{name:"class transformers.ElectraForSequenceClassification",anchor:"transformers.ElectraForSequenceClassification",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L795",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.ElectraForSequenceClassification.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForSequenceClassification">ElectraForSequenceClassification</a>) — | |
| Model configuration class with all the parameters of the model. Initializing with a config file does not | |
| load the weights associated with the model, only the configuration. Check out the | |
| <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var le=e(ie,8),ce=s(le);l(ce,{name:"forward",anchor:"transformers.ElectraForSequenceClassification.forward",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L806",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"labels",val:": typing.Optional[torch.Tensor] = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.ElectraForSequenceClassification.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p> | |
| <p>Indices can be obtained using <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and | |
| <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.ElectraForSequenceClassification.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.ElectraForSequenceClassification.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 corresponds to a <em>sentence A</em> token,</li> | |
| <li>1 corresponds to a <em>sentence B</em> token.</li> | |
| </ul> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"token_type_ids"},{anchor:"transformers.ElectraForSequenceClassification.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p> | |
| <p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.ElectraForSequenceClassification.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This | |
| is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the | |
| model’s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.ElectraForSequenceClassification.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) — | |
| Labels for computing the sequence classification/regression loss. Indices should be in <code>[0, ..., config.num_labels - 1]</code>. If <code>config.num_labels == 1</code> a regression loss is computed (Mean-Square loss), If | |
| <code>config.num_labels > 1</code> a classification loss is computed (Cross-Entropy).`,name:"labels"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.SequenceClassifierOutput" | |
| >SequenceClassifierOutput</a> or a tuple of | |
| <code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various | |
| elements depending on the configuration (<a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraConfig" | |
| >ElectraConfig</a>) and inputs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.SequenceClassifierOutput" | |
| >SequenceClassifierOutput</a> or <code>tuple(torch.FloatTensor)</code></p> | |
| `});var de=e(ce,4);h(de,{children:(t,a)=>{var o=u();i(t,o)},$$slots:{default:!0}});var pe=e(de,4);m(pe,{anchor:"transformers.ElectraForSequenceClassification.forward.example",children:(t,a)=>{var o=no(),r=e(p(o),2);d(r,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEVsZWN0cmFGb3JTZXF1ZW5jZUNsYXNzaWZpY2F0aW9uJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZWxlY3RyYS1zbWFsbC1kaXNjcmltaW5hdG9yJTIyKSUwQW1vZGVsJTIwJTNEJTIwRWxlY3RyYUZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtc21hbGwtZGlzY3JpbWluYXRvciUyMiklMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIySGVsbG8lMkMlMjBteSUyMGRvZyUyMGlzJTIwY3V0ZSUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMGxvZ2l0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKS5sb2dpdHMlMEElMEFwcmVkaWN0ZWRfY2xhc3NfaWQlMjAlM0QlMjBsb2dpdHMuYXJnbWF4KCkuaXRlbSgpJTBBbW9kZWwuY29uZmlnLmlkMmxhYmVsJTVCcHJlZGljdGVkX2NsYXNzX2lkJTVEJTBBJTBBJTIzJTIwVG8lMjB0cmFpbiUyMGElMjBtb2RlbCUyMG9uJTIwJTYwbnVtX2xhYmVscyU2MCUyMGNsYXNzZXMlMkMlMjB5b3UlMjBjYW4lMjBwYXNzJTIwJTYwbnVtX2xhYmVscyUzRG51bV9sYWJlbHMlNjAlMjB0byUyMCU2MC5mcm9tX3ByZXRyYWluZWQoLi4uKSU2MCUwQW51bV9sYWJlbHMlMjAlM0QlMjBsZW4obW9kZWwuY29uZmlnLmlkMmxhYmVsKSUwQW1vZGVsJTIwJTNEJTIwRWxlY3RyYUZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtc21hbGwtZGlzY3JpbWluYXRvciUyMiUyQyUyMG51bV9sYWJlbHMlM0RudW1fbGFiZWxzKSUwQSUwQWxhYmVscyUyMCUzRCUyMHRvcmNoLnRlbnNvciglNUIxJTVEKSUwQWxvc3MlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyUyQyUyMGxhYmVscyUzRGxhYmVscykubG9zcyUwQXJvdW5kKGxvc3MuaXRlbSgpJTJDJTIwMik=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, ElectraForSequenceClassification | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>model = ElectraForSequenceClassification.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>inputs = tokenizer(<span class="hljs-string">"Hello, my dog is cute"</span>, return_tensors=<span class="hljs-string">"pt"</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad(): | |
| <span class="hljs-meta">... </span> logits = model(**inputs).logits | |
| <span class="hljs-meta">>>> </span>predicted_class_id = logits.argmax().item() | |
| <span class="hljs-meta">>>> </span>model.config.id2label[predicted_class_id] | |
| ... | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># To train a model on \`num_labels\` classes, you can pass \`num_labels=num_labels\` to \`.from_pretrained(...)\`</span> | |
| <span class="hljs-meta">>>> </span>num_labels = <span class="hljs-built_in">len</span>(model.config.id2label) | |
| <span class="hljs-meta">>>> </span>model = ElectraForSequenceClassification.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>, num_labels=num_labels) | |
| <span class="hljs-meta">>>> </span>labels = torch.tensor([<span class="hljs-number">1</span>]) | |
| <span class="hljs-meta">>>> </span>loss = model(**inputs, labels=labels).loss | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">round</span>(loss.item(), <span class="hljs-number">2</span>) | |
| ...`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}});var Re=e(pe,2);m(Re,{anchor:"transformers.ElectraForSequenceClassification.forward.example-2",children:(t,a)=>{var o=ao(),r=e(p(o),2);d(r,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEVsZWN0cmFGb3JTZXF1ZW5jZUNsYXNzaWZpY2F0aW9uJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZWxlY3RyYS1zbWFsbC1kaXNjcmltaW5hdG9yJTIyKSUwQW1vZGVsJTIwJTNEJTIwRWxlY3RyYUZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtc21hbGwtZGlzY3JpbWluYXRvciUyMiUyQyUyMHByb2JsZW1fdHlwZSUzRCUyMm11bHRpX2xhYmVsX2NsYXNzaWZpY2F0aW9uJTIyKSUwQSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJIZWxsbyUyQyUyMG15JTIwZG9nJTIwaXMlMjBjdXRlJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwbG9naXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpLmxvZ2l0cyUwQSUwQXByZWRpY3RlZF9jbGFzc19pZHMlMjAlM0QlMjB0b3JjaC5hcmFuZ2UoMCUyQyUyMGxvZ2l0cy5zaGFwZSU1Qi0xJTVEKSU1QnRvcmNoLnNpZ21vaWQobG9naXRzKS5zcXVlZXplKGRpbSUzRDApJTIwJTNFJTIwMC41JTVEJTBBJTBBJTIzJTIwVG8lMjB0cmFpbiUyMGElMjBtb2RlbCUyMG9uJTIwJTYwbnVtX2xhYmVscyU2MCUyMGNsYXNzZXMlMkMlMjB5b3UlMjBjYW4lMjBwYXNzJTIwJTYwbnVtX2xhYmVscyUzRG51bV9sYWJlbHMlNjAlMjB0byUyMCU2MC5mcm9tX3ByZXRyYWluZWQoLi4uKSU2MCUwQW51bV9sYWJlbHMlMjAlM0QlMjBsZW4obW9kZWwuY29uZmlnLmlkMmxhYmVsKSUwQW1vZGVsJTIwJTNEJTIwRWxlY3RyYUZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmdvb2dsZSUyRmVsZWN0cmEtc21hbGwtZGlzY3JpbWluYXRvciUyMiUyQyUyMG51bV9sYWJlbHMlM0RudW1fbGFiZWxzJTJDJTIwcHJvYmxlbV90eXBlJTNEJTIybXVsdGlfbGFiZWxfY2xhc3NpZmljYXRpb24lMjIlMEEpJTBBJTBBbGFiZWxzJTIwJTNEJTIwdG9yY2guc3VtKCUwQSUyMCUyMCUyMCUyMHRvcmNoLm5uLmZ1bmN0aW9uYWwub25lX2hvdChwcmVkaWN0ZWRfY2xhc3NfaWRzJTVCTm9uZSUyQyUyMCUzQSU1RC5jbG9uZSgpJTJDJTIwbnVtX2NsYXNzZXMlM0RudW1fbGFiZWxzKSUyQyUyMGRpbSUzRDElMEEpLnRvKHRvcmNoLmZsb2F0KSUwQWxvc3MlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyUyQyUyMGxhYmVscyUzRGxhYmVscykubG9zcw==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, ElectraForSequenceClassification | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>model = ElectraForSequenceClassification.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>, problem_type=<span class="hljs-string">"multi_label_classification"</span>) | |
| <span class="hljs-meta">>>> </span>inputs = tokenizer(<span class="hljs-string">"Hello, my dog is cute"</span>, return_tensors=<span class="hljs-string">"pt"</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad(): | |
| <span class="hljs-meta">... </span> logits = model(**inputs).logits | |
| <span class="hljs-meta">>>> </span>predicted_class_ids = torch.arange(<span class="hljs-number">0</span>, logits.shape[-<span class="hljs-number">1</span>])[torch.sigmoid(logits).squeeze(dim=<span class="hljs-number">0</span>) > <span class="hljs-number">0.5</span>] | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># To train a model on \`num_labels\` classes, you can pass \`num_labels=num_labels\` to \`.from_pretrained(...)\`</span> | |
| <span class="hljs-meta">>>> </span>num_labels = <span class="hljs-built_in">len</span>(model.config.id2label) | |
| <span class="hljs-meta">>>> </span>model = ElectraForSequenceClassification.from_pretrained( | |
| <span class="hljs-meta">... </span> <span class="hljs-string">"google/electra-small-discriminator"</span>, num_labels=num_labels, problem_type=<span class="hljs-string">"multi_label_classification"</span> | |
| <span class="hljs-meta">... </span>) | |
| <span class="hljs-meta">>>> </span>labels = torch.<span class="hljs-built_in">sum</span>( | |
| <span class="hljs-meta">... </span> torch.nn.functional.one_hot(predicted_class_ids[<span class="hljs-literal">None</span>, :].clone(), num_classes=num_labels), dim=<span class="hljs-number">1</span> | |
| <span class="hljs-meta">... </span>).to(torch.<span class="hljs-built_in">float</span>) | |
| <span class="hljs-meta">>>> </span>loss = model(**inputs, labels=labels).loss`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}}),n(le),n(J);var me=e(J,2);c(me,{title:"ElectraForMultipleChoice",local:"transformers.ElectraForMultipleChoice",headingTag:"h2"});var U=e(me,2),he=s(U);l(he,{name:"class transformers.ElectraForMultipleChoice",anchor:"transformers.ElectraForMultipleChoice",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L1152",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.ElectraForMultipleChoice.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForMultipleChoice">ElectraForMultipleChoice</a>) — | |
| Model configuration class with all the parameters of the model. Initializing with a config file does not | |
| load the weights associated with the model, only the configuration. Check out the | |
| <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var ue=e(he,8),ge=s(ue);l(ge,{name:"forward",anchor:"transformers.ElectraForMultipleChoice.forward",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L1163",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"labels",val:": typing.Optional[torch.Tensor] = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.ElectraForMultipleChoice.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, num_choices, sequence_length)</code>) — | |
| Indices of input sequence tokens in the vocabulary.</p> | |
| <p>Indices can be obtained using <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and | |
| <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.ElectraForMultipleChoice.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.ElectraForMultipleChoice.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, num_choices, sequence_length)</code>, <em>optional</em>) — | |
| Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 corresponds to a <em>sentence A</em> token,</li> | |
| <li>1 corresponds to a <em>sentence B</em> token.</li> | |
| </ul> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"token_type_ids"},{anchor:"transformers.ElectraForMultipleChoice.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, num_choices, sequence_length)</code>, <em>optional</em>) — | |
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.max_position_embeddings - 1]</code>.</p> | |
| <p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.ElectraForMultipleChoice.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_choices, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This | |
| is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the | |
| model’s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.ElectraForMultipleChoice.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) — | |
| Labels for computing the multiple choice classification loss. Indices should be in <code>[0, ..., num_choices-1]</code> where <code>num_choices</code> is the size of the second dimension of the input tensors. (See | |
| <code>input_ids</code> above)`,name:"labels"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.MultipleChoiceModelOutput" | |
| >MultipleChoiceModelOutput</a> or a tuple of | |
| <code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various | |
| elements depending on the configuration (<a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraConfig" | |
| >ElectraConfig</a>) and inputs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.MultipleChoiceModelOutput" | |
| >MultipleChoiceModelOutput</a> or <code>tuple(torch.FloatTensor)</code></p> | |
| `});var fe=e(ge,4);h(fe,{children:(t,a)=>{var o=u();i(t,o)},$$slots:{default:!0}});var Ne=e(fe,4);m(Ne,{anchor:"transformers.ElectraForMultipleChoice.forward.example",children:(t,a)=>{var o=_(),r=e(p(o),2);d(r,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBFbGVjdHJhRm9yTXVsdGlwbGVDaG9pY2UlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtc21hbGwtZGlzY3JpbWluYXRvciUyMiklMEFtb2RlbCUyMCUzRCUyMEVsZWN0cmFGb3JNdWx0aXBsZUNob2ljZS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZWxlY3RyYS1zbWFsbC1kaXNjcmltaW5hdG9yJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkluJTIwSXRhbHklMkMlMjBwaXp6YSUyMHNlcnZlZCUyMGluJTIwZm9ybWFsJTIwc2V0dGluZ3MlMkMlMjBzdWNoJTIwYXMlMjBhdCUyMGElMjByZXN0YXVyYW50JTJDJTIwaXMlMjBwcmVzZW50ZWQlMjB1bnNsaWNlZC4lMjIlMEFjaG9pY2UwJTIwJTNEJTIwJTIySXQlMjBpcyUyMGVhdGVuJTIwd2l0aCUyMGElMjBmb3JrJTIwYW5kJTIwYSUyMGtuaWZlLiUyMiUwQWNob2ljZTElMjAlM0QlMjAlMjJJdCUyMGlzJTIwZWF0ZW4lMjB3aGlsZSUyMGhlbGQlMjBpbiUyMHRoZSUyMGhhbmQuJTIyJTBBbGFiZWxzJTIwJTNEJTIwdG9yY2gudGVuc29yKDApLnVuc3F1ZWV6ZSgwKSUyMCUyMCUyMyUyMGNob2ljZTAlMjBpcyUyMGNvcnJlY3QlMjAoYWNjb3JkaW5nJTIwdG8lMjBXaWtpcGVkaWElMjAlM0IpKSUyQyUyMGJhdGNoJTIwc2l6ZSUyMDElMEElMEFlbmNvZGluZyUyMCUzRCUyMHRva2VuaXplciglNUJwcm9tcHQlMkMlMjBwcm9tcHQlNUQlMkMlMjAlNUJjaG9pY2UwJTJDJTIwY2hvaWNlMSU1RCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMkMlMjBwYWRkaW5nJTNEVHJ1ZSklMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKiolN0JrJTNBJTIwdi51bnNxdWVlemUoMCklMjBmb3IlMjBrJTJDJTIwdiUyMGluJTIwZW5jb2RpbmcuaXRlbXMoKSU3RCUyQyUyMGxhYmVscyUzRGxhYmVscyklMjAlMjAlMjMlMjBiYXRjaCUyMHNpemUlMjBpcyUyMDElMEElMEElMjMlMjB0aGUlMjBsaW5lYXIlMjBjbGFzc2lmaWVyJTIwc3RpbGwlMjBuZWVkcyUyMHRvJTIwYmUlMjB0cmFpbmVkJTBBbG9zcyUyMCUzRCUyMG91dHB1dHMubG9zcyUwQWxvZ2l0cyUyMCUzRCUyMG91dHB1dHMubG9naXRz",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, ElectraForMultipleChoice | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>model = ElectraForMultipleChoice.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>prompt = <span class="hljs-string">"In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."</span> | |
| <span class="hljs-meta">>>> </span>choice0 = <span class="hljs-string">"It is eaten with a fork and a knife."</span> | |
| <span class="hljs-meta">>>> </span>choice1 = <span class="hljs-string">"It is eaten while held in the hand."</span> | |
| <span class="hljs-meta">>>> </span>labels = torch.tensor(<span class="hljs-number">0</span>).unsqueeze(<span class="hljs-number">0</span>) <span class="hljs-comment"># choice0 is correct (according to Wikipedia ;)), batch size 1</span> | |
| <span class="hljs-meta">>>> </span>encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors=<span class="hljs-string">"pt"</span>, padding=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>outputs = model(**{k: v.unsqueeze(<span class="hljs-number">0</span>) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoding.items()}, labels=labels) <span class="hljs-comment"># batch size is 1</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># the linear classifier still needs to be trained</span> | |
| <span class="hljs-meta">>>> </span>loss = outputs.loss | |
| <span class="hljs-meta">>>> </span>logits = outputs.logits`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}}),n(ue),n(U);var _e=e(U,2);c(_e,{title:"ElectraForTokenClassification",local:"transformers.ElectraForTokenClassification",headingTag:"h2"});var E=e(_e,2),be=s(E);l(be,{name:"class transformers.ElectraForTokenClassification",anchor:"transformers.ElectraForTokenClassification",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L1027",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.ElectraForTokenClassification.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForTokenClassification">ElectraForTokenClassification</a>) — | |
| Model configuration class with all the parameters of the model. Initializing with a config file does not | |
| load the weights associated with the model, only the configuration. Check out the | |
| <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var ye=e(be,10),Te=s(ye);l(Te,{name:"forward",anchor:"transformers.ElectraForTokenClassification.forward",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L1041",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"labels",val:": typing.Optional[torch.Tensor] = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.ElectraForTokenClassification.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p> | |
| <p>Indices can be obtained using <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and | |
| <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.ElectraForTokenClassification.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.ElectraForTokenClassification.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 corresponds to a <em>sentence A</em> token,</li> | |
| <li>1 corresponds to a <em>sentence B</em> token.</li> | |
| </ul> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"token_type_ids"},{anchor:"transformers.ElectraForTokenClassification.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p> | |
| <p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.ElectraForTokenClassification.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This | |
| is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the | |
| model’s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.ElectraForTokenClassification.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Labels for computing the token classification loss. Indices should be in <code>[0, ..., config.num_labels - 1]</code>.`,name:"labels"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.TokenClassifierOutput" | |
| >TokenClassifierOutput</a> or a tuple of | |
| <code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various | |
| elements depending on the configuration (<a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraConfig" | |
| >ElectraConfig</a>) and inputs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.TokenClassifierOutput" | |
| >TokenClassifierOutput</a> or <code>tuple(torch.FloatTensor)</code></p> | |
| `});var Me=e(Te,4);h(Me,{children:(t,a)=>{var o=u();i(t,o)},$$slots:{default:!0}});var Ve=e(Me,4);m(Ve,{anchor:"transformers.ElectraForTokenClassification.forward.example",children:(t,a)=>{var o=_(),r=e(p(o),2);d(r,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBFbGVjdHJhRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQWltcG9ydCUyMHRvcmNoJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZWxlY3RyYS1zbWFsbC1kaXNjcmltaW5hdG9yJTIyKSUwQW1vZGVsJTIwJTNEJTIwRWxlY3RyYUZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtc21hbGwtZGlzY3JpbWluYXRvciUyMiklMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTBBJTIwJTIwJTIwJTIwJTIySHVnZ2luZ0ZhY2UlMjBpcyUyMGElMjBjb21wYW55JTIwYmFzZWQlMjBpbiUyMFBhcmlzJTIwYW5kJTIwTmV3JTIwWW9yayUyMiUyQyUyMGFkZF9zcGVjaWFsX3Rva2VucyUzREZhbHNlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUwQSklMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwbG9naXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpLmxvZ2l0cyUwQSUwQXByZWRpY3RlZF90b2tlbl9jbGFzc19pZHMlMjAlM0QlMjBsb2dpdHMuYXJnbWF4KC0xKSUwQSUwQSUyMyUyME5vdGUlMjB0aGF0JTIwdG9rZW5zJTIwYXJlJTIwY2xhc3NpZmllZCUyMHJhdGhlciUyMHRoZW4lMjBpbnB1dCUyMHdvcmRzJTIwd2hpY2glMjBtZWFucyUyMHRoYXQlMEElMjMlMjB0aGVyZSUyMG1pZ2h0JTIwYmUlMjBtb3JlJTIwcHJlZGljdGVkJTIwdG9rZW4lMjBjbGFzc2VzJTIwdGhhbiUyMHdvcmRzLiUwQSUyMyUyME11bHRpcGxlJTIwdG9rZW4lMjBjbGFzc2VzJTIwbWlnaHQlMjBhY2NvdW50JTIwZm9yJTIwdGhlJTIwc2FtZSUyMHdvcmQlMEFwcmVkaWN0ZWRfdG9rZW5zX2NsYXNzZXMlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0Lml0ZW0oKSU1RCUyMGZvciUyMHQlMjBpbiUyMHByZWRpY3RlZF90b2tlbl9jbGFzc19pZHMlNUIwJTVEJTVEJTBBcHJlZGljdGVkX3Rva2Vuc19jbGFzc2VzJTBBJTBBbGFiZWxzJTIwJTNEJTIwcHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyUwQWxvc3MlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyUyQyUyMGxhYmVscyUzRGxhYmVscykubG9zcyUwQXJvdW5kKGxvc3MuaXRlbSgpJTJDJTIwMik=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, ElectraForTokenClassification | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>model = ElectraForTokenClassification.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>inputs = tokenizer( | |
| <span class="hljs-meta">... </span> <span class="hljs-string">"HuggingFace is a company based in Paris and New York"</span>, add_special_tokens=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">"pt"</span> | |
| <span class="hljs-meta">... </span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad(): | |
| <span class="hljs-meta">... </span> logits = model(**inputs).logits | |
| <span class="hljs-meta">>>> </span>predicted_token_class_ids = logits.argmax(-<span class="hljs-number">1</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Note that tokens are classified rather then input words which means that</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># there might be more predicted token classes than words.</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Multiple token classes might account for the same word</span> | |
| <span class="hljs-meta">>>> </span>predicted_tokens_classes = [model.config.id2label[t.item()] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predicted_token_class_ids[<span class="hljs-number">0</span>]] | |
| <span class="hljs-meta">>>> </span>predicted_tokens_classes | |
| ... | |
| <span class="hljs-meta">>>> </span>labels = predicted_token_class_ids | |
| <span class="hljs-meta">>>> </span>loss = model(**inputs, labels=labels).loss | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">round</span>(loss.item(), <span class="hljs-number">2</span>) | |
| ...`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}}),n(ye),n(E);var ke=e(E,2);c(ke,{title:"ElectraForQuestionAnswering",local:"transformers.ElectraForQuestionAnswering",headingTag:"h2"});var z=e(ke,2),we=s(z);l(we,{name:"class transformers.ElectraForQuestionAnswering",anchor:"transformers.ElectraForQuestionAnswering",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L1081",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.ElectraForQuestionAnswering.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraForQuestionAnswering">ElectraForQuestionAnswering</a>) — | |
| Model configuration class with all the parameters of the model. Initializing with a config file does not | |
| load the weights associated with the model, only the configuration. Check out the | |
| <a href="/docs/transformers/pr_46007/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var ve=e(we,8),je=s(ve);l(je,{name:"forward",anchor:"transformers.ElectraForQuestionAnswering.forward",source:"https://github.com/huggingface/transformers/blob/vr_46007/src/transformers/models/electra/modeling_electra.py#L1095",parameters:[{name:"input_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"token_type_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.Tensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"start_positions",val:": typing.Optional[torch.Tensor] = None"},{name:"end_positions",val:": typing.Optional[torch.Tensor] = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.ElectraForQuestionAnswering.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.</p> | |
| <p>Indices can be obtained using <a href="/docs/transformers/pr_46007/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and | |
| <a href="/docs/transformers/pr_46007/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p> | |
| <p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.ElectraForQuestionAnswering.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>1 for tokens that are <strong>not masked</strong>,</li> | |
| <li>0 for tokens that are <strong>masked</strong>.</li> | |
| </ul> | |
| <p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.ElectraForQuestionAnswering.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p> | |
| <ul> | |
| <li>0 corresponds to a <em>sentence A</em> token,</li> | |
| <li>1 corresponds to a <em>sentence B</em> token.</li> | |
| </ul> | |
| <p><a href="../glossary#token-type-ids">What are token type IDs?</a>`,name:"token_type_ids"},{anchor:"transformers.ElectraForQuestionAnswering.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) — | |
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p> | |
| <p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.ElectraForQuestionAnswering.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) — | |
| Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This | |
| is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the | |
| model’s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.ElectraForQuestionAnswering.forward.start_positions",description:`<strong>start_positions</strong> (<code>torch.Tensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) — | |
| Labels for position (index) of the start of the labelled span for computing the token classification loss. | |
| Positions are clamped to the length of the sequence (<code>sequence_length</code>). Position outside of the sequence | |
| are not taken into account for computing the loss.`,name:"start_positions"},{anchor:"transformers.ElectraForQuestionAnswering.forward.end_positions",description:`<strong>end_positions</strong> (<code>torch.Tensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) — | |
| Labels for position (index) of the end of the labelled span for computing the token classification loss. | |
| Positions are clamped to the length of the sequence (<code>sequence_length</code>). Position outside of the sequence | |
| are not taken into account for computing the loss.`,name:"end_positions"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A <a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.QuestionAnsweringModelOutput" | |
| >QuestionAnsweringModelOutput</a> or a tuple of | |
| <code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various | |
| elements depending on the configuration (<a | |
| href="/docs/transformers/pr_46007/en/model_doc/electra#transformers.ElectraConfig" | |
| >ElectraConfig</a>) and inputs.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><a | |
| href="/docs/transformers/pr_46007/en/main_classes/output#transformers.modeling_outputs.QuestionAnsweringModelOutput" | |
| >QuestionAnsweringModelOutput</a> or <code>tuple(torch.FloatTensor)</code></p> | |
| `});var Je=e(je,4);h(Je,{children:(t,a)=>{var o=u();i(t,o)},$$slots:{default:!0}});var Xe=e(Je,4);m(Xe,{anchor:"transformers.ElectraForQuestionAnswering.forward.example",children:(t,a)=>{var o=_(),r=e(p(o),2);d(r,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBFbGVjdHJhRm9yUXVlc3Rpb25BbnN3ZXJpbmclMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmVsZWN0cmEtc21hbGwtZGlzY3JpbWluYXRvciUyMiklMEFtb2RlbCUyMCUzRCUyMEVsZWN0cmFGb3JRdWVzdGlvbkFuc3dlcmluZy5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZWxlY3RyYS1zbWFsbC1kaXNjcmltaW5hdG9yJTIyKSUwQSUwQXF1ZXN0aW9uJTJDJTIwdGV4dCUyMCUzRCUyMCUyMldobyUyMHdhcyUyMEppbSUyMEhlbnNvbiUzRiUyMiUyQyUyMCUyMkppbSUyMEhlbnNvbiUyMHdhcyUyMGElMjBuaWNlJTIwcHVwcGV0JTIyJTBBJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHF1ZXN0aW9uJTJDJTIwdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEElMEFhbnN3ZXJfc3RhcnRfaW5kZXglMjAlM0QlMjBvdXRwdXRzLnN0YXJ0X2xvZ2l0cy5hcmdtYXgoKSUwQWFuc3dlcl9lbmRfaW5kZXglMjAlM0QlMjBvdXRwdXRzLmVuZF9sb2dpdHMuYXJnbWF4KCklMEElMEFwcmVkaWN0X2Fuc3dlcl90b2tlbnMlMjAlM0QlMjBpbnB1dHMuaW5wdXRfaWRzJTVCMCUyQyUyMGFuc3dlcl9zdGFydF9pbmRleCUyMCUzQSUyMGFuc3dlcl9lbmRfaW5kZXglMjAlMkIlMjAxJTVEJTBBdG9rZW5pemVyLmRlY29kZShwcmVkaWN0X2Fuc3dlcl90b2tlbnMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklMEElMEElMjMlMjB0YXJnZXQlMjBpcyUyMCUyMm5pY2UlMjBwdXBwZXQlMjIlMEF0YXJnZXRfc3RhcnRfaW5kZXglMjAlM0QlMjB0b3JjaC50ZW5zb3IoJTVCMTQlNUQpJTBBdGFyZ2V0X2VuZF9pbmRleCUyMCUzRCUyMHRvcmNoLnRlbnNvciglNUIxNSU1RCklMEElMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMlMkMlMjBzdGFydF9wb3NpdGlvbnMlM0R0YXJnZXRfc3RhcnRfaW5kZXglMkMlMjBlbmRfcG9zaXRpb25zJTNEdGFyZ2V0X2VuZF9pbmRleCklMEFsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3NzJTBBcm91bmQobG9zcy5pdGVtKCklMkMlMjAyKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, ElectraForQuestionAnswering | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>model = ElectraForQuestionAnswering.from_pretrained(<span class="hljs-string">"google/electra-small-discriminator"</span>) | |
| <span class="hljs-meta">>>> </span>question, text = <span class="hljs-string">"Who was Jim Henson?"</span>, <span class="hljs-string">"Jim Henson was a nice puppet"</span> | |
| <span class="hljs-meta">>>> </span>inputs = tokenizer(question, text, return_tensors=<span class="hljs-string">"pt"</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad(): | |
| <span class="hljs-meta">... </span> outputs = model(**inputs) | |
| <span class="hljs-meta">>>> </span>answer_start_index = outputs.start_logits.argmax() | |
| <span class="hljs-meta">>>> </span>answer_end_index = outputs.end_logits.argmax() | |
| <span class="hljs-meta">>>> </span>predict_answer_tokens = inputs.input_ids[<span class="hljs-number">0</span>, answer_start_index : answer_end_index + <span class="hljs-number">1</span>] | |
| <span class="hljs-meta">>>> </span>tokenizer.decode(predict_answer_tokens, skip_special_tokens=<span class="hljs-literal">True</span>) | |
| ... | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># target is "nice puppet"</span> | |
| <span class="hljs-meta">>>> </span>target_start_index = torch.tensor([<span class="hljs-number">14</span>]) | |
| <span class="hljs-meta">>>> </span>target_end_index = torch.tensor([<span class="hljs-number">15</span>]) | |
| <span class="hljs-meta">>>> </span>outputs = model(**inputs, start_positions=target_start_index, end_positions=target_end_index) | |
| <span class="hljs-meta">>>> </span>loss = outputs.loss | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">round</span>(loss.item(), <span class="hljs-number">2</span>) | |
| ...`,lang:"python",wrap:!1}),i(t,o)},$$slots:{default:!0}}),n(ve),n(z);var Se=e(z,2);Oe(Se,{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/electra.md"}),f(2),i(ze,F),eo()}export{uo as component}; | |
Xet Storage Details
- Size:
- 140 kB
- Xet hash:
- fde4552a6f0372632f3a71c1e6268d446f142233eb68011c397d74770c737c3a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.