onnx/tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": {
    "strategy": "BatchLongest",
    "direction": "Right",
    "pad_to_multiple_of": null,
    "pad_id": 2,
    "pad_type_id": 0,
    "pad_token": "<pad>"
  },
  "added_tokens": [
    {
      "id": 0,
      "content": "<s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 1,
      "content": "</s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 2,
      "content": "<pad>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 3,
      "content": "<mask>",
      "single_word": false,
      "lstrip": true,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 4,
      "content": "<unk>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "ByteLevel",
    "add_prefix_space": false,
    "trim_offsets": true,
    "use_regex": true
  },
  "post_processor": {
    "type": "RobertaProcessing",
    "sep": [
      "</s>",
      1
    ],
    "cls": [
      "<s>",
      0
    ],
    "trim_offsets": true,
    "add_prefix_space": false
  },
  "decoder": {
    "type": "ByteLevel",
    "add_prefix_space": true,
    "trim_offsets": true,
    "use_regex": true
  },
  "model": {
    "type": "BPE",
    "dropout": null,
    "unk_token": null,
    "continuing_subword_prefix": "",
    "end_of_word_suffix": "",
    "fuse_unk": false,
    "byte_fallback": false,
    "ignore_merges": false,
    "vocab": {
      "<s>": 0,
      "</s>": 1,
      "<pad>": 2,
      "<mask>": 3,
      "<unk>": 4,
      "c": 5,
      "C": 6,
      ">1": 7,
      "O": 8,
      "=": 9,
      "N": 10,
      "^5": 11,
      "n": 12,
      "^4": 13,
      ">2": 14,
      "[C@@H]": 15,
      "[C@H]": 16,
      "^6": 17,
      "F": 18,
      "S": 19,
      "Cl": 20,
      "[NH+]": 21,
      "s": 22,
      "-": 23,
      "o": 24,
      "^8": 25,
      "^9": 26,
      "^7": 27,
      ">3": 28,
      "[O-]": 29,
      "[nH]": 30,
      "[NH2+]": 31,
      ">4": 32,
      ">7": 33,
      "#": 34,
      "^2": 35,
      ">8": 36,
      ">6": 37,
      "^10": 38,
      "Br": 39,
      ">9": 40,
      "[N+]": 41,
      "%T": 42,
      ">10": 43,
      ">11": 44,
      ">5": 45,
      ">12": 46,
      ">13": 47,
      ">14": 48,
      "^11": 49,
      ">15": 50,
      "[NH3+]": 51,
      "[C@]": 52,
      "[C@@]": 53,
      "^12": 54,
      "^13": 55,
      "%C": 56,
      "^16": 57,
      ">16": 58,
      "^15": 59,
      "^17": 60,
      "^14": 61,
      "^18": 62,
      "^19": 63,
      ">17": 64,
      "^20": 65,
      "^21": 66,
      ">18": 67,
      "^22": 68,
      "^3": 69,
      "^23": 70,
      ">19": 71,
      "^24": 72,
      ">20": 73,
      "^25": 74,
      "[N-]": 75,
      "[S@@]": 76,
      ">21": 77,
      "[S@]": 78,
      "^26": 79,
      "I": 80,
      ">22": 81,
      "^27": 82,
      "=^8": 83,
      "=^6": 84,
      "[S-]": 85,
      "^28": 86,
      ">23": 87,
      "^29": 88,
      ">24": 89,
      "=^5": 90,
      "=^4": 91,
      "^30": 92,
      ">25": 93,
      "^31": 94,
      "=^11": 95,
      "=^9": 96,
      "=^10": 97,
      "=^7": 98,
      ">26": 99,
      "^32": 100,
      "=^12": 101,
      "P": 102,
      "-^9": 103,
      ">27": 104,
      "^33": 105,
      "=^13": 106,
      "=^14": 107,
      "=^15": 108,
      "=^16": 109,
      "=^17": 110,
      ">28": 111,
      "-^11": 112,
      "[P@]": 113,
      "=^18": 114,
      ">29": 115,
      "[P@@]": 116,
      "^34": 117,
      "=^19": 118,
      "-^4": 119,
      "=^20": 120,
      "=^21": 121,
      "=^23": 122,
      "=^22": 123,
      "-^10": 124,
      "=^25": 125,
      "=^28": 126,
      "[O+]": 127,
      "[OH+]": 128,
      "-^12": 129,
      "-^18": 130,
      "-^5": 131,
      "-^7": 132,
      "=^24": 133,
      "=^26": 134,
      "[CH-]": 135,
      "[NH-]": 136,
      "[S+]": 137,
      "[SH+]": 138,
      "^35": 139,
      "-^13": 140,
      "-^14": 141,
      "-^17": 142,
      "-^20": 143,
      "-^23": 144,
      "-^6": 145,
      "=^27": 146,
      ">30": 147,
      "[CH2-]": 148,
      "[P+]": 149,
      "[P@@H]": 150
    },
    "merges": []
  }
}