morph_tokenizer/tokenizer_config.json

{
  "tokenizer_type": "morpheme_v4",
  "special_tokens": [
    "[PAD]",
    "[UNK]",
    "[CLS]",
    "[SEP]",
    "[MASK]"
  ],
  "pad_id": 0,
  "unk_id": 1,
  "cls_id": 2,
  "sep_id": 3,
  "mask_id": 4,
  "vocab_size": 57029,
  "max_length": 128,
  "min_freq_word": 2,
  "min_freq_morph": 3,
  "seg_temp_train": 1.0,
  "seg_temp_eval": 0.0,
  "model_dims": {
    "d_model": 512,
    "n_heads": 8,
    "n_layers": 6,
    "d_ff": 2048,
    "dropout": 0.1
  }
}