Upload tokenizer

Browse files

Files changed (3) hide show

special_tokens_map.json +9 -0
tokenizer.json +540 -0
tokenizer_config.json +5 -0

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+  "bos_token": "<cls>",
+  "cls_token": "<cls>",
+  "eos_token": "<sep>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "<sep>",
+  "unk_token": "<unk>"
+}

tokenizer.json ADDED Viewed

	@@ -0,0 +1,540 @@

+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<cls>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<sep>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "<mask>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "<cls>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "<cls>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "<cls>": {
+        "id": "<cls>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "<cls>"
+        ]
+      },
+      "<sep>": {
+        "id": "<sep>",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "<sep>"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "add_prefix_space": true
+  },
+  "model": {
+    "type": "Unigram",
+    "unk_id": 3,
+    "vocab": [
+      [
+        "<pad>",
+        0.0
+      ],
+      [
+        "<cls>",
+        0.0
+      ],
+      [
+        "<sep>",
+        0.0
+      ],
+      [
+        "<unk>",
+        0.0
+      ],
+      [
+        "<mask>",
+        0.0
+      ],
+      [
+        "K",
+        -2.8926598139924984
+      ],
+      [
+        "P",
+        -2.937301100450547
+      ],
+      [
+        "R",
+        -2.981133587027994
+      ],
+      [
+        "N",
+        -2.984398973523902
+      ],
+      [
+        "F",
+        -3.0405482512940942
+      ],
+      [
+        "Q",
+        -3.0522167753465475
+      ],
+      [
+        "T",
+        -3.057893521866589
+      ],
+      [
+        "D",
+        -3.221572989674069
+      ],
+      [
+        "V",
+        -3.242957729449966
+      ],
+      [
+        "I",
+        -3.2639228157131353
+      ],
+      [
+        "G",
+        -3.292222729720054
+      ],
+      [
+        "Y",
+        -3.3242430893206976
+      ],
+      [
+        "E",
+        -3.329998422834164
+      ],
+      [
+        "A",
+        -3.3315100842234955
+      ],
+      [
+        "L",
+        -3.354133164459414
+      ],
+      [
+        "S",
+        -3.374619245946816
+      ],
+      [
+        "H",
+        -3.6029119248880974
+      ],
+      [
+        "M",
+        -3.6330880431340073
+      ],
+      [
+        "C",
+        -4.03859206801765
+      ],
+      [
+        "W",
+        -4.123791187360837
+      ],
+      [
+        "AA",
+        -4.973167425197216
+      ],
+      [
+        "LL",
+        -5.036049835340242
+      ],
+      [
+        "LA",
+        -5.194188976627833
+      ],
+      [
+        "LS",
+        -5.284020121682021
+      ],
+      [
+        "AL",
+        -5.320469682162143
+      ],
+      [
+        "SS",
+        -5.32453022570105
+      ],
+      [
+        "SL",
+        -5.499862731096803
+      ],
+      [
+        "LV",
+        -5.564795933199976
+      ],
+      [
+        "AG",
+        -5.570161883025232
+      ],
+      [
+        "GG",
+        -5.586688874755673
+      ],
+      [
+        "AV",
+        -5.633866882153034
+      ],
+      [
+        "LG",
+        -5.6617838822297415
+      ],
+      [
+        "EL",
+        -5.710679099236735
+      ],
+      [
+        "AS",
+        -5.716264001404388
+      ],
+      [
+        "VL",
+        -5.7256437689651545
+      ],
+      [
+        "SG",
+        -5.727475385041664
+      ],
+      [
+        "LR",
+        -5.728549131116866
+      ],
+      [
+        "LE",
+        -5.743482374609412
+      ],
+      [
+        "GL",
+        -5.743495521291919
+      ],
+      [
+        "SA",
+        -5.761398906433778
+      ],
+      [
+        "GA",
+        -5.779501575037608
+      ],
+      [
+        "EE",
+        -5.783251373522621
+      ],
+      [
+        "RL",
+        -5.787802487132645
+      ],
+      [
+        "VA",
+        -5.822397606320607
+      ],
+      [
+        "TL",
+        -5.826633127578567
+      ],
+      [
+        "GS",
+        -5.836663706756646
+      ],
+      [
+        "EA",
+        -5.854012392063035
+      ],
+      [
+        "TA",
+        -5.87623583133286
+      ],
+      [
+        "LD",
+        -5.884057317313408
+      ],
+      [
+        "VV",
+        -5.886122471861352
+      ],
+      [
+        "LT",
+        -5.890501009220628
+      ],
+      [
+        "IL",
+        -5.893641541215938
+      ],
+      [
+        "DL",
+        -5.908589432452223
+      ],
+      [
+        "SV",
+        -5.928508582466128
+      ],
+      [
+        "LP",
+        -5.930607331306064
+      ],
+      [
+        "GV",
+        -5.931353514901755
+      ],
+      [
+        "IS",
+        -5.934577159636316
+      ],
+      [
+        "RR",
+        -5.937181139038799
+      ],
+      [
+        "PA",
+        -5.95165892093522
+      ],
+      [
+        "LI",
+        -5.956444805023796
+      ],
+      [
+        "DG",
+        -5.959872500616557
+      ],
+      [
+        "LK",
+        -5.977419747714027
+      ],
+      [
+        "AE",
+        -5.988346330589572
+      ],
+      [
+        "VS",
+        -5.992644676581763
+      ],
+      [
+        "TV",
+        -5.992771047038934
+      ],
+      [
+        "TS",
+        -5.992862744926201
+      ],
+      [
+        "DA",
+        -6.005980312620514
+      ],
+      [
+        "TG",
+        -6.020054424800335
+      ],
+      [
+        "IA",
+        -6.029275865888938
+      ],
+      [
+        "PS",
+        -6.0558861763675385
+      ],
+      [
+        "RA",
+        -6.091301969244897
+      ],
+      [
+        "EK",
+        -6.143807832748726
+      ],
+      [
+        "EV",
+        -6.144172113809702
+      ],
+      [
+        "ST",
+        -6.1659183042917824
+      ],
+      [
+        "DE",
+        -6.175007806118581
+      ],
+      [
+        "SI",
+        -6.196141783532548
+      ],
+      [
+        "IV",
+        -6.203941668151083
+      ],
+      [
+        "DV",
+        -6.2062585412907065
+      ],
+      [
+        "AD",
+        -6.214482574668974
+      ],
+      [
+        "RS",
+        -6.219463683778585
+      ],
+      [
+        "EI",
+        -6.2248807356059555
+      ],
+      [
+        "GE",
+        -6.253174496738863
+      ],
+      [
+        "IE",
+        -6.264016428440042
+      ],
+      [
+        "ES",
+        -6.272022775598392
+      ],
+      [
+        "SE",
+        -6.288530115689479
+      ],
+      [
+        "IG",
+        -6.300749800545876
+      ],
+      [
+        "SD",
+        -6.305378344633647
+      ],
+      [
+        "VG",
+        -6.307254140261982
+      ],
+      [
+        "VE",
+        -6.322491387400987
+      ],
+      [
+        "GI",
+        -6.346224238884332
+      ],
+      [
+        "X",
+        -7.588889207971681
+      ],
+      [
+        "B",
+        -14.434103952435104
+      ],
+      [
+        "Z",
+        -15.661068824545652
+      ],
+      [
+        "U",
+        -16.34051994314413
+      ],
+      [
+        "O",
+        -19.83562802135349
+      ]
+    ]
+  }
+}

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,5 @@

+{
+  "clean_up_tokenization_spaces": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}