| from transformers.configuration_utils import PretrainedConfig |
|
|
| class STLConfig(PretrainedConfig): |
|
|
| model_type = "stldec32" |
| keys_to_ignore_at_inference = ["past_key_values"] |
| attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"} |
|
|
| def __init__( |
| self, |
| vocab_size=35, |
| decoder_vocab_size=None, |
| max_position_embeddings=512, |
| encoder_layers=12, |
| encoder_ffn_dim=4096, |
| encoder_attention_heads=16, |
| decoder_layers=12, |
| decoder_ffn_dim=4096, |
| decoder_attention_heads=16, |
| encoder_layerdrop=0.0, |
| decoder_layerdrop=0.0, |
| use_cache=True, |
| is_encoder_decoder=True, |
| activation_function="gelu", |
| d_model=32, |
| dropout=0.1, |
| attention_dropout=0.0, |
| activation_dropout=0.0, |
| init_std=0.02, |
| decoder_start_token_id=3, |
| scale_embedding=False, |
| pad_token_id=1, |
| eos_token_id=3, |
| bos_token_id=2, |
| forced_eos_token_id=3, |
| share_encoder_decoder_embeddings=True, |
| **kwargs, |
| ): |
| self.vocab_size = vocab_size |
| self.decoder_vocab_size = decoder_vocab_size or vocab_size |
| self.max_position_embeddings = max_position_embeddings |
| self.d_model = d_model |
| self.encoder_ffn_dim = encoder_ffn_dim |
| self.encoder_layers = encoder_layers |
| self.encoder_attention_heads = encoder_attention_heads |
| self.decoder_ffn_dim = decoder_ffn_dim |
| self.decoder_layers = decoder_layers |
| self.decoder_attention_heads = decoder_attention_heads |
| self.dropout = dropout |
| self.attention_dropout = attention_dropout |
| self.activation_dropout = activation_dropout |
| self.activation_function = activation_function |
| self.init_std = init_std |
| self.encoder_layerdrop = encoder_layerdrop |
| self.decoder_layerdrop = decoder_layerdrop |
| self.use_cache = use_cache |
| self.num_hidden_layers = encoder_layers |
| self.scale_embedding = scale_embedding |
| self.share_encoder_decoder_embeddings = share_encoder_decoder_embeddings |
| super().__init__( |
| bos_token_id=bos_token_id, |
| pad_token_id=pad_token_id, |
| eos_token_id=eos_token_id, |
| is_encoder_decoder=is_encoder_decoder, |
| decoder_start_token_id=decoder_start_token_id, |
| forced_eos_token_id=forced_eos_token_id, |
| **kwargs, |
| ) |
|
|
