Soham Jain
Initial commit of GPT-2 46M SwiGLU Dense baseline model architecture and configs
83e4eda verified | from transformers import PretrainedConfig | |
| class GPT2CustomConfig(PretrainedConfig): | |
| model_type = "gpt2_custom" | |
| auto_map = { | |
| "AutoConfig": "configuration_gpt2.GPT2CustomConfig", | |
| "AutoModelForCausalLM": "modeling_gpt2.GPT2CustomLMHeadModel" | |
| } | |
| def __init__( | |
| self, | |
| vocab_size=16384, | |
| n_positions=512, | |
| n_embd=512, | |
| n_layer=12, | |
| n_head=8, | |
| n_inner=1360, | |
| activation_function="swiglu", | |
| resid_pdrop=0.1, | |
| embd_pdrop=0.1, | |
| attn_pdrop=0.1, | |
| layer_norm_epsilon=1e-5, | |
| initializer_range=0.02, | |
| bos_token_id=0, | |
| eos_token_id=0, | |
| **kwargs | |
| ): | |
| super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) | |
| self.vocab_size = vocab_size | |
| self.n_positions = n_positions | |
| self.n_embd = n_embd | |
| self.n_layer = n_layer | |
| self.n_head = n_head | |
| self.n_inner = n_inner | |
| self.activation_function = activation_function | |
| self.resid_pdrop = resid_pdrop | |
| self.embd_pdrop = embd_pdrop | |
| self.attn_pdrop = attn_pdrop | |
| self.layer_norm_epsilon = layer_norm_epsilon | |
| self.initializer_range = initializer_range | |