Upload tokenizer

added_tokens.json

@@ -1,5 +1,4 @@
 {
-  "</s>": 90,
-  "<s>": 89,
-  "|": 91
+  "</s>": 92,
+  "<s>": 91
 }

tokenizer_config.json

@@ -1,6 +1,14 @@
 {
   "added_tokens_decoder": {
-    "87": {
+    "55": {
+      "content": "|",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "89": {
       "content": "[UNK]",
       "lstrip": true,
       "normalized": false,
@@ -8,7 +16,7 @@
       "single_word": false,
       "special": false
     },
-    "88": {
+    "90": {
       "content": "[PAD]",
       "lstrip": true,
       "normalized": false,
@@ -16,7 +24,7 @@
       "single_word": false,
       "special": false
     },
-    "89": {
+    "91": {
       "content": "<s>",
       "lstrip": false,
       "normalized": false,
@@ -24,21 +32,13 @@
       "single_word": false,
       "special": true
     },
-    "90": {
+    "92": {
       "content": "</s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "91": {
-      "content": "|",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
   "additional_special_tokens": null,

vocab.json

@@ -1,91 +1,93 @@
 {
-  " ": 0,
-  "[PAD]": 88,
-  "[UNK]": 87,
-  "々": 1,
-  "ぁ": 2,
-  "あ": 3,
-  "ぃ": 4,
-  "い": 5,
-  "ぅ": 6,
-  "う": 7,
-  "ぇ": 8,
-  "え": 9,
-  "ぉ": 10,
-  "お": 11,
-  "か": 12,
-  "が": 13,
-  "き": 14,
-  "ぎ": 15,
-  "く": 16,
-  "ぐ": 17,
-  "け": 18,
-  "げ": 19,
-  "こ": 20,
-  "ご": 21,
-  "さ": 22,
-  "ざ": 23,
-  "し": 24,
-  "じ": 25,
-  "す": 26,
-  "ず": 27,
-  "せ": 28,
-  "ぜ": 29,
-  "そ": 30,
-  "ぞ": 31,
-  "た": 32,
-  "だ": 33,
-  "ち": 34,
-  "ぢ": 35,
-  "っ": 36,
-  "つ": 37,
-  "づ": 38,
-  "て": 39,
-  "で": 40,
-  "と": 41,
-  "ど": 42,
-  "な": 43,
+  "[PAD]": 90,
+  "[UNK]": 89,
+  "|": 55,
+  "々": 81,
+  "ぁ": 74,
+  "あ": 64,
+  "ぃ": 78,
+  "い": 61,
+  "ぅ": 7,
+  "う": 58,
+  "ぇ": 20,
+  "え": 59,
+  "ぉ": 49,
+  "お": 60,
+  "か": 18,
+  "が": 46,
+  "き": 57,
+  "ぎ": 70,
+  "く": 75,
+  "ぐ": 80,
+  "け": 35,
+  "げ": 15,
+  "こ": 73,
+  "ご": 27,
+  "さ": 10,
+  "ざ": 16,
+  "し": 72,
+  "じ": 2,
+  "す": 67,
+  "ず": 26,
+  "せ": 4,
+  "ぜ": 77,
+  "そ": 56,
+  "ぞ": 54,
+  "た": 39,
+  "だ": 1,
+  "ち": 23,
+  "ぢ": 5,
+  "っ": 69,
+  "つ": 22,
+  "づ": 11,
+  "て": 76,
+  "で": 85,
+  "と": 24,
+  "ど": 66,
+  "な": 25,
   "に": 44,
-  "ぬ": 45,
-  "ね": 46,
-  "の": 47,
-  "は": 48,
-  "ば": 49,
-  "ぱ": 50,
-  "ひ": 51,
-  "び": 52,
-  "ぴ": 53,
-  "ふ": 54,
-  "ぶ": 55,
-  "ぷ": 56,
-  "へ": 57,
-  "べ": 58,
-  "ぺ": 59,
-  "ほ": 60,
-  "ぼ": 61,
-  "ぽ": 62,
-  "ま": 63,
-  "み": 64,
-  "む": 65,
-  "め": 66,
-  "も": 67,
+  "ぬ": 37,
+  "ね": 28,
+  "の": 51,
+  "は": 34,
+  "ば": 43,
+  "ぱ": 29,
+  "ひ": 8,
+  "び": 19,
+  "ぴ": 62,
+  "ふ": 53,
+  "ぶ": 50,
+  "ぷ": 45,
+  "へ": 9,
+  "べ": 42,
+  "ぺ": 3,
+  "ほ": 31,
+  "ぼ": 33,
+  "ぽ": 65,
+  "ま": 0,
+  "み": 40,
+  "む": 41,
+  "め": 36,
+  "も": 6,
   "ゃ": 68,
-  "や": 69,
-  "ゅ": 70,
-  "ゆ": 71,
-  "ょ": 72,
-  "よ": 73,
-  "ら": 74,
-  "り": 75,
-  "る": 76,
-  "れ": 77,
-  "ろ": 78,
-  "ゎ": 79,
-  "わ": 80,
-  "を": 81,
-  "ん": 82,
-  "ゔ": 83,
-  "ゕ": 84,
-  "ゖ": 85,
-  "ゝ": 86
+  "や": 52,
+  "ゅ": 21,
+  "ゆ": 32,
+  "ょ": 17,
+  "よ": 84,
+  "ら": 88,
+  "り": 30,
+  "る": 79,
+  "れ": 38,
+  "ろ": 14,
+  "ゎ": 82,
+  "わ": 87,
+  "ゐ": 13,
+  "を": 86,
+  "ん": 83,
+  "ゔ": 12,
+  "ゕ": 48,
+  "ゖ": 63,
+  "ゝ": 71,
+  "ゞ": 47
 }