| | from dataclasses import dataclass
|
| | from typing import Optional, Tuple
|
| |
|
| | import torch
|
| | import torch.nn as nn
|
| | from transformers import ErnieModel, ErniePreTrainedModel, PretrainedConfig
|
| | from transformers.file_utils import ModelOutput
|
| |
|
| | from .decode_utils import UIEDecoder
|
| |
|
| |
|
| | @dataclass
|
| | class UIEModelOutput(ModelOutput):
|
| | """
|
| | Output class for outputs of UIE.
|
| | losses (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
|
| | Total spn extraction losses is the sum of a Cross-Entropy for the start and end positions.
|
| | start_prob (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
|
| | Span-start scores (after Sigmoid).
|
| | end_prob (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
|
| | Span-end scores (after Sigmoid).
|
| | hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
|
| | Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layers, +
|
| | one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
|
| | Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
|
| | attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
|
| | Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| | sequence_length)`.
|
| | Attention weights after the attention softmax, used to compute the weighted average in the self-attention
|
| | heads.
|
| | """
|
| | loss: Optional[torch.FloatTensor] = None
|
| | start_prob: torch.FloatTensor = None
|
| | end_prob: torch.FloatTensor = None
|
| | start_positions: torch.FloatTensor = None
|
| | end_positions: torch.FloatTensor = None
|
| | hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| | attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| |
|
| |
|
| | class UIEModel(ErniePreTrainedModel, UIEDecoder):
|
| | """
|
| | UIE model based on Bert model.
|
| | This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
| | library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
| | etc.)
|
| | This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
| | Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
| | and behavior.
|
| | Parameters:
|
| | config ([`PretrainedConfig`]): Model configuration class with all the parameters of the model.
|
| | Initializing with a config file does not load the weights associated with the model, only the
|
| | configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
|
| | """
|
| |
|
| | def __init__(self, config: PretrainedConfig):
|
| | super(UIEModel, self).__init__(config)
|
| | self.encoder = ErnieModel(config)
|
| | self.config = config
|
| | hidden_size = self.config.hidden_size
|
| |
|
| | self.linear_start = nn.Linear(hidden_size, 1)
|
| | self.linear_end = nn.Linear(hidden_size, 1)
|
| | self.sigmoid = nn.Sigmoid()
|
| |
|
| | self.post_init()
|
| |
|
| | def forward(
|
| | self,
|
| | input_ids: Optional[torch.Tensor] = None,
|
| | token_type_ids: Optional[torch.Tensor] = None,
|
| | position_ids: Optional[torch.Tensor] = None,
|
| | attention_mask: Optional[torch.Tensor] = None,
|
| | head_mask: Optional[torch.Tensor] = None,
|
| | inputs_embeds: Optional[torch.Tensor] = None,
|
| | start_positions: Optional[torch.Tensor] = None,
|
| | end_positions: Optional[torch.Tensor] = None,
|
| | output_attentions: Optional[bool] = None,
|
| | output_hidden_states: Optional[bool] = None,
|
| | ) -> UIEModelOutput:
|
| | """
|
| | Args:
|
| | input_ids (`torch.LongTensor` of shape `({0})`):
|
| | Indices of input sequence tokens in the vocabulary.
|
| | Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
| | [`PreTrainedTokenizer.__call__`] for details.
|
| | [What are input IDs?](../glossary#input-ids)
|
| | attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
|
| | Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| | - 1 for tokens that are **not masked**,
|
| | - 0 for tokens that are **masked**.
|
| | [What are attention masks?](../glossary#attention-mask)
|
| | token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
|
| | Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
|
| | 1]`:
|
| | - 0 corresponds to a *sentence A* token,
|
| | - 1 corresponds to a *sentence B* token.
|
| | [What are token type IDs?](../glossary#token-type-ids)
|
| | position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
|
| | Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
|
| | config.max_position_embeddings - 1]`.
|
| | [What are position IDs?](../glossary#position-ids)
|
| | head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
|
| | Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
|
| | - 1 indicates the head is **not masked**,
|
| | - 0 indicates the head is **masked**.
|
| | inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
|
| | Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
|
| | is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
|
| | model's internal embedding lookup matrix.
|
| | start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
| | Labels for position (index) of the start of the labelled span for computing the token classification loss.
|
| | Positions are clamped to the length of the sequence (`sequence_length`). Position outsides of the sequence
|
| | are not taken into account for computing the loss.
|
| | end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
| | Labels for position (index) of the end of the labelled span for computing the token classification loss.
|
| | Positions are clamped to the length of the sequence (`sequence_length`). Position outsides of the sequence
|
| | are not taken into account for computing the loss.
|
| | output_attentions (`bool`, *optional*):
|
| | Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
| | tensors for more detail.
|
| | output_hidden_states (`bool`, *optional*):
|
| | Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
| | more detail.
|
| | return_dict (`bool`, *optional*):
|
| | Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| | """
|
| | outputs = self.encoder(
|
| | input_ids=input_ids,
|
| | token_type_ids=token_type_ids,
|
| | position_ids=position_ids,
|
| | attention_mask=attention_mask,
|
| | head_mask=head_mask,
|
| | inputs_embeds=inputs_embeds,
|
| | output_attentions=output_attentions,
|
| | output_hidden_states=output_hidden_states,
|
| | )
|
| | sequence_output = outputs[0]
|
| |
|
| | start_logits = self.linear_start(sequence_output)
|
| | start_logits = torch.squeeze(start_logits, -1)
|
| | start_prob = self.sigmoid(start_logits)
|
| |
|
| | end_logits = self.linear_end(sequence_output)
|
| | end_logits = torch.squeeze(end_logits, -1)
|
| | end_prob = self.sigmoid(end_logits)
|
| |
|
| | total_loss = None
|
| | if start_positions is not None and end_positions is not None:
|
| | loss_fct = nn.BCELoss()
|
| | start_loss = loss_fct(start_prob, start_positions)
|
| | end_loss = loss_fct(end_prob, end_positions)
|
| |
|
| | total_loss = (start_loss + end_loss) / 2.0
|
| |
|
| | return UIEModelOutput(
|
| | loss=total_loss,
|
| | start_prob=start_prob,
|
| | end_prob=end_prob,
|
| | hidden_states=outputs.hidden_states,
|
| | attentions=outputs.attentions,
|
| | )
|
| |
|
