tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "<pad>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 1,
      "content": "<cls>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 2,
      "content": "<sep>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 3,
      "content": "<unk>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 4,
      "content": "<mask>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": null,
  "post_processor": {
    "type": "TemplateProcessing",
    "single": [
      {
        "SpecialToken": {
          "id": "<cls>",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      },
      {
        "SpecialToken": {
          "id": "<sep>",
          "type_id": 0
        }
      }
    ],
    "pair": [
      {
        "SpecialToken": {
          "id": "<cls>",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      },
      {
        "SpecialToken": {
          "id": "<sep>",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "B",
          "type_id": 1
        }
      },
      {
        "SpecialToken": {
          "id": "<sep>",
          "type_id": 1
        }
      }
    ],
    "special_tokens": {
      "<cls>": {
        "id": "<cls>",
        "ids": [
          1
        ],
        "tokens": [
          "<cls>"
        ]
      },
      "<sep>": {
        "id": "<sep>",
        "ids": [
          2
        ],
        "tokens": [
          "<sep>"
        ]
      }
    }
  },
  "decoder": {
    "type": "Metaspace",
    "replacement": "▁",
    "add_prefix_space": true
  },
  "model": {
    "type": "Unigram",
    "unk_id": 3,
    "vocab": [
      [
        "<pad>",
        0.0
      ],
      [
        "<cls>",
        0.0
      ],
      [
        "<sep>",
        0.0
      ],
      [
        "<unk>",
        0.0
      ],
      [
        "<mask>",
        0.0
      ],
      [
        "K",
        -2.8926598139924984
      ],
      [
        "P",
        -2.937301100450547
      ],
      [
        "R",
        -2.981133587027994
      ],
      [
        "N",
        -2.984398973523902
      ],
      [
        "F",
        -3.0405482512940942
      ],
      [
        "Q",
        -3.0522167753465475
      ],
      [
        "T",
        -3.057893521866589
      ],
      [
        "D",
        -3.221572989674069
      ],
      [
        "V",
        -3.242957729449966
      ],
      [
        "I",
        -3.2639228157131353
      ],
      [
        "G",
        -3.292222729720054
      ],
      [
        "Y",
        -3.3242430893206976
      ],
      [
        "E",
        -3.329998422834164
      ],
      [
        "A",
        -3.3315100842234955
      ],
      [
        "L",
        -3.354133164459414
      ],
      [
        "S",
        -3.374619245946816
      ],
      [
        "H",
        -3.6029119248880974
      ],
      [
        "M",
        -3.6330880431340073
      ],
      [
        "C",
        -4.03859206801765
      ],
      [
        "W",
        -4.123791187360837
      ],
      [
        "AA",
        -4.973167425197216
      ],
      [
        "LL",
        -5.036049835340242
      ],
      [
        "LA",
        -5.194188976627833
      ],
      [
        "LS",
        -5.284020121682021
      ],
      [
        "AL",
        -5.320469682162143
      ],
      [
        "SS",
        -5.32453022570105
      ],
      [
        "SL",
        -5.499862731096803
      ],
      [
        "LV",
        -5.564795933199976
      ],
      [
        "AG",
        -5.570161883025232
      ],
      [
        "GG",
        -5.586688874755673
      ],
      [
        "AV",
        -5.633866882153034
      ],
      [
        "LG",
        -5.6617838822297415
      ],
      [
        "EL",
        -5.710679099236735
      ],
      [
        "AS",
        -5.716264001404388
      ],
      [
        "VL",
        -5.7256437689651545
      ],
      [
        "SG",
        -5.727475385041664
      ],
      [
        "LR",
        -5.728549131116866
      ],
      [
        "LE",
        -5.743482374609412
      ],
      [
        "GL",
        -5.743495521291919
      ],
      [
        "SA",
        -5.761398906433778
      ],
      [
        "GA",
        -5.779501575037608
      ],
      [
        "EE",
        -5.783251373522621
      ],
      [
        "RL",
        -5.787802487132645
      ],
      [
        "VA",
        -5.822397606320607
      ],
      [
        "TL",
        -5.826633127578567
      ],
      [
        "GS",
        -5.836663706756646
      ],
      [
        "EA",
        -5.854012392063035
      ],
      [
        "TA",
        -5.87623583133286
      ],
      [
        "LD",
        -5.884057317313408
      ],
      [
        "VV",
        -5.886122471861352
      ],
      [
        "LT",
        -5.890501009220628
      ],
      [
        "IL",
        -5.893641541215938
      ],
      [
        "DL",
        -5.908589432452223
      ],
      [
        "SV",
        -5.928508582466128
      ],
      [
        "LP",
        -5.930607331306064
      ],
      [
        "GV",
        -5.931353514901755
      ],
      [
        "IS",
        -5.934577159636316
      ],
      [
        "RR",
        -5.937181139038799
      ],
      [
        "PA",
        -5.95165892093522
      ],
      [
        "LI",
        -5.956444805023796
      ],
      [
        "DG",
        -5.959872500616557
      ],
      [
        "LK",
        -5.977419747714027
      ],
      [
        "AE",
        -5.988346330589572
      ],
      [
        "VS",
        -5.992644676581763
      ],
      [
        "TV",
        -5.992771047038934
      ],
      [
        "TS",
        -5.992862744926201
      ],
      [
        "DA",
        -6.005980312620514
      ],
      [
        "TG",
        -6.020054424800335
      ],
      [
        "IA",
        -6.029275865888938
      ],
      [
        "PS",
        -6.0558861763675385
      ],
      [
        "RA",
        -6.091301969244897
      ],
      [
        "EK",
        -6.143807832748726
      ],
      [
        "EV",
        -6.144172113809702
      ],
      [
        "ST",
        -6.1659183042917824
      ],
      [
        "DE",
        -6.175007806118581
      ],
      [
        "SI",
        -6.196141783532548
      ],
      [
        "IV",
        -6.203941668151083
      ],
      [
        "DV",
        -6.2062585412907065
      ],
      [
        "AD",
        -6.214482574668974
      ],
      [
        "RS",
        -6.219463683778585
      ],
      [
        "EI",
        -6.2248807356059555
      ],
      [
        "GE",
        -6.253174496738863
      ],
      [
        "IE",
        -6.264016428440042
      ],
      [
        "ES",
        -6.272022775598392
      ],
      [
        "SE",
        -6.288530115689479
      ],
      [
        "IG",
        -6.300749800545876
      ],
      [
        "SD",
        -6.305378344633647
      ],
      [
        "VG",
        -6.307254140261982
      ],
      [
        "VE",
        -6.322491387400987
      ],
      [
        "GI",
        -6.346224238884332
      ],
      [
        "X",
        -7.588889207971681
      ],
      [
        "B",
        -14.434103952435104
      ],
      [
        "Z",
        -15.661068824545652
      ],
      [
        "U",
        -16.34051994314413
      ],
      [
        "O",
        -19.83562802135349
      ]
    ]
  }
}