File size: 1,376 Bytes
3bfb9a0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | from transformers import BertConfig
class DualBertConfig(BertConfig):
model_type = "dual_bert"
def __init__(
self,
vocab_char_size: int = 256,
vocab_word_size: int = 50000,
word_char_emb_dim: int = 192,
hidden_size: int = 512,
num_hidden_layers: int = 6,
num_attention_heads: int = 8,
intermediate_size: int = 2048,
max_position_embeddings: int = 512,
hidden_dropout_prob: float = 0.1,
attention_probs_dropout_prob: float = 0.1,
**kwargs,
):
# HuggingFace passes vocab_size via kwargs when loading from config.json;
# remove it to avoid conflict with our explicit vocab_size=vocab_char_size.
kwargs.pop("vocab_size", None)
super().__init__(
vocab_size=vocab_char_size,
hidden_size=hidden_size,
num_hidden_layers=num_hidden_layers,
num_attention_heads=num_attention_heads,
intermediate_size=intermediate_size,
max_position_embeddings=max_position_embeddings,
hidden_dropout_prob=hidden_dropout_prob,
attention_probs_dropout_prob=attention_probs_dropout_prob,
**kwargs,
)
self.vocab_char_size = vocab_char_size
self.vocab_word_size = vocab_word_size
self.word_char_emb_dim = word_char_emb_dim |