|
|
"""HELM-BERT configuration.""" |
|
|
|
|
|
from transformers import PretrainedConfig |
|
|
|
|
|
|
|
|
class HELMBertConfig(PretrainedConfig): |
|
|
"""Configuration class for HELM-BERT model. |
|
|
|
|
|
This configuration class stores all the parameters needed to instantiate a HELM-BERT model. |
|
|
It inherits from PretrainedConfig and can be used with HuggingFace's from_pretrained and |
|
|
save_pretrained methods. |
|
|
|
|
|
Args: |
|
|
vocab_size: Size of the vocabulary (default: 78 for HELM character vocabulary) |
|
|
hidden_size: Dimensionality of the encoder layers (default: 768) |
|
|
num_hidden_layers: Number of transformer layers (default: 6) |
|
|
num_attention_heads: Number of attention heads (default: 12) |
|
|
intermediate_size: Dimensionality of the feed-forward layer (default: 3072) |
|
|
hidden_dropout_prob: Dropout probability for hidden layers (default: 0.1) |
|
|
attention_probs_dropout_prob: Dropout probability for attention (default: 0.1) |
|
|
max_position_embeddings: Maximum sequence length (default: 512) |
|
|
max_relative_positions: Maximum relative position distance (default: 512) |
|
|
position_buckets: Number of position buckets for log-bucketing (default: 256) |
|
|
pos_att_type: Position attention types, pipe-separated (default: "c2p|p2c") |
|
|
share_att_key: Whether to share attention key projections (default: False) |
|
|
ngie_kernel_size: Kernel size for nGiE convolution (default: 3) |
|
|
ngie_dropout: Dropout for nGiE layer (default: 0.1) |
|
|
pad_token_id: ID of padding token (default: 0) |
|
|
bos_token_id: ID of beginning-of-sequence token (default: 1) |
|
|
eos_token_id: ID of end-of-sequence token (default: 2) |
|
|
sep_token_id: ID of separator token (default: 2) |
|
|
mask_token_id: ID of mask token (default: 4) |
|
|
|
|
|
Example: |
|
|
>>> from helmbert import HELMBertConfig, HELMBertModel |
|
|
>>> config = HELMBertConfig(hidden_size=768, num_hidden_layers=6) |
|
|
>>> model = HELMBertModel(config) |
|
|
""" |
|
|
|
|
|
model_type = "helmbert" |
|
|
|
|
|
def __init__( |
|
|
self, |
|
|
vocab_size: int = 78, |
|
|
hidden_size: int = 768, |
|
|
num_hidden_layers: int = 6, |
|
|
num_attention_heads: int = 12, |
|
|
intermediate_size: int = 3072, |
|
|
hidden_dropout_prob: float = 0.1, |
|
|
attention_probs_dropout_prob: float = 0.1, |
|
|
max_position_embeddings: int = 512, |
|
|
|
|
|
max_relative_positions: int = 512, |
|
|
position_buckets: int = 256, |
|
|
pos_att_type: str = "c2p|p2c", |
|
|
share_att_key: bool = False, |
|
|
|
|
|
ngie_kernel_size: int = 3, |
|
|
ngie_dropout: float = 0.1, |
|
|
|
|
|
pad_token_id: int = 0, |
|
|
bos_token_id: int = 1, |
|
|
eos_token_id: int = 2, |
|
|
sep_token_id: int = 2, |
|
|
mask_token_id: int = 4, |
|
|
|
|
|
num_labels: int = 2, |
|
|
problem_type: str = None, |
|
|
classifier_num_layers: int = 0, |
|
|
classifier_dropout: float = 0.1, |
|
|
**kwargs, |
|
|
): |
|
|
super().__init__( |
|
|
pad_token_id=pad_token_id, |
|
|
bos_token_id=bos_token_id, |
|
|
eos_token_id=eos_token_id, |
|
|
**kwargs, |
|
|
) |
|
|
|
|
|
|
|
|
self.vocab_size = vocab_size |
|
|
self.hidden_size = hidden_size |
|
|
self.num_hidden_layers = num_hidden_layers |
|
|
self.num_attention_heads = num_attention_heads |
|
|
self.intermediate_size = intermediate_size |
|
|
self.hidden_dropout_prob = hidden_dropout_prob |
|
|
self.attention_probs_dropout_prob = attention_probs_dropout_prob |
|
|
self.max_position_embeddings = max_position_embeddings |
|
|
|
|
|
|
|
|
self.max_relative_positions = max_relative_positions |
|
|
self.position_buckets = position_buckets |
|
|
self.pos_att_type = pos_att_type |
|
|
self.share_att_key = share_att_key |
|
|
|
|
|
|
|
|
self.ngie_kernel_size = ngie_kernel_size |
|
|
self.ngie_dropout = ngie_dropout |
|
|
|
|
|
|
|
|
self.sep_token_id = sep_token_id |
|
|
self.mask_token_id = mask_token_id |
|
|
|
|
|
|
|
|
self.num_labels = num_labels |
|
|
self.problem_type = problem_type |
|
|
self.classifier_num_layers = classifier_num_layers |
|
|
self.classifier_dropout = classifier_dropout |
|
|
|
