config.json

{"vocab_size": 50304, "n_positions": 1024, "n_ctx": 1024, "n_embd": 768, "n_layer": 12, "n_head": 12, "activation_function": "gelu", "resid_pdrop": 0.1, "attn_pdrop": 0.1, "summary_type": "cls_index"}