config.json

{
  "vocab_size": 73448,
  "pad_token_id": 0,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "plh_token_id": 73446,
  "embedding_dim": 1024,
  "d_model": 512,
  "n_heads": 8,
  "d_ff": 2048,
  "n_encoder_layers": 6,
  "n_decoder_layers": 6,
  "dropout": 0.0,
  "attention_dropout": 0.0,
  "activation": "gelu",
  "pos_encoding_type": "alibi",
  "rope_base": 10000.0,
  "max_source_positions": 1024,
  "max_target_positions": 1024,
  "qk_norm": true,
  "headwise_attn_output_gate": false,
  "elementwise_attn_output_gate": false,
  "max_placeholder": 255,
  "early_exit_del": null,
  "early_exit_plh": null
}