tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "<unk>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 67,
      "content": "<s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 68,
      "content": "</s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 69,
      "content": "<pad>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 70,
      "content": "<mask>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": {
    "type": "NFKC"
  },
  "pre_tokenizer": {
    "type": "Metaspace",
    "replacement": "▁",
    "add_prefix_space": true,
    "prepend_scheme": "always"
  },
  "post_processor": null,
  "decoder": {
    "type": "Metaspace",
    "replacement": "▁",
    "add_prefix_space": true,
    "prepend_scheme": "always"
  },
  "model": {
    "type": "BPE",
    "dropout": null,
    "unk_token": "<unk>",
    "continuing_subword_prefix": null,
    "end_of_word_suffix": null,
    "fuse_unk": false,
    "byte_fallback": false,
    "vocab": {
      "<unk>": 0,
      "ං": 1,
      "අ": 2,
      "ආ": 3,
      "ඇ": 4,
      "ඈ": 5,
      "ඉ": 6,
      "ඊ": 7,
      "උ": 8,
      "ඌ": 9,
      "එ": 10,
      "ඒ": 11,
      "ඓ": 12,
      "ඔ": 13,
      "ඕ": 14,
      "ක": 15,
      "ඛ": 16,
      "ග": 17,
      "ඝ": 18,
      "ඟ": 19,
      "ච": 20,
      "ඡ": 21,
      "ජ": 22,
      "ඤ": 23,
      "ට": 24,
      "ඨ": 25,
      "ඩ": 26,
      "ඪ": 27,
      "ණ": 28,
      "ඬ": 29,
      "ත": 30,
      "ථ": 31,
      "ද": 32,
      "ධ": 33,
      "න": 34,
      "ඳ": 35,
      "ප": 36,
      "ඵ": 37,
      "බ": 38,
      "භ": 39,
      "ම": 40,
      "ඹ": 41,
      "ය": 42,
      "ර": 43,
      "ල": 44,
      "ව": 45,
      "ශ": 46,
      "ෂ": 47,
      "ස": 48,
      "හ": 49,
      "ළ": 50,
      "ෆ": 51,
      "්": 52,
      "ා": 53,
      "ැ": 54,
      "ෑ": 55,
      "ි": 56,
      "ී": 57,
      "ු": 58,
      "ූ": 59,
      "ෙ": 60,
      "ේ": 61,
      "ෛ": 62,
      "ො": 63,
      "ෝ": 64,
      "ෞ": 65,
      "▁": 66
    },
    "merges": []
  }
}