divilian
/

lilchatbot-tokenizer

Model card Files Files and versions

xet

Community

divilian commited on Feb 2

Commit

c1d4b22

verified ·

1 Parent(s): b36d6fa

Upload tokenizer.json with huggingface_hub

Browse files

Files changed (1) hide show

tokenizer.json +102 -109

tokenizer.json CHANGED Viewed

@@ -5,15 +5,6 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "<unk>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 1,
       "content": "<pad>",
       "single_word": false,
       "lstrip": false,
@@ -22,7 +13,7 @@
       "special": true
     },
     {
-      "id": 29,
       "content": "<bos>",
       "single_word": false,
       "lstrip": false,
@@ -31,7 +22,7 @@
       "special": true
     },
     {
-      "id": 30,
       "content": "<eos>",
       "single_word": false,
       "lstrip": false,
@@ -93,104 +84,106 @@
   "model": {
     "type": "WordLevel",
     "vocab": {
-      "<unk>": 0,
-      "<pad>": 1,
-      ",": 4,
-      "the": 5,
-      ".": 6,
-      "of": 7,
-      "and": 8,
-      "to": 9,
-      "a": 10,
-      "in": 11,
-      "\"": 12,
-      "that": 13,
-      "was": 14,
-      "is": 15,
-      "for": 16,
-      "on": 17,
-      "with": 18,
-      "as": 19,
-      "it": 20,
-      "he": 21,
-      "his": 22,
-      "at": 23,
-      "by": 24,
-      "i": 25,
-      ";": 26,
-      "be": 27,
-      "from": 28,
-      "<bos>": 29,
-      "<eos>": 30,
-      "not": 31,
-      ")": 32,
-      "but": 33,
-      "(": 34,
-      "this": 35,
-      "had": 36,
-      "which": 37,
-      "have": 38,
-      "are": 39,
-      ":": 40,
-      "they": 41,
-      "an": 42,
-      "or": 43,
-      "were": 44,
-      "her": 45,
-      "said": 46,
-      "their": 47,
-      "has": 48,
-      "we": 49,
-      "all": 50,
-      "who": 51,
-      "one": 52,
-      "she": 53,
-      "you": 54,
-      "“": 55,
-      "”": 56,
-      "been": 57,
-      "will": 58,
-      "more": 59,
-      "him": 60,
-      "when": 61,
-      "so": 62,
-      "its": 63,
-      "would": 64,
-      "there": 65,
-      "no": 66,
-      "after": 67,
-      "my": 68,
-      "if": 69,
-      "out": 70,
-      "also": 71,
-      "about": 72,
-      "them": 73,
-      "up": 74,
-      "into": 75,
-      "two": 76,
-      "what": 77,
-      "new": 78,
-      "first": 79,
-      "some": 80,
-      "can": 81,
-      "time": 82,
-      "than": 83,
-      "other": 84,
-      "our": 85,
-      "me": 86,
-      "only": 87,
-      "over": 88,
-      "could": 89,
-      "may": 90,
-      "?": 91,
-      "any": 92,
-      "/": 93,
-      "then": 94,
-      "people": 95,
-      "like": 96,
-      "now": 97,
-      "such": 98,
-      "do": 99
     },
     "unk_token": "<unk>"
   }

   "added_tokens": [
     {
       "id": 0,
       "content": "<pad>",
       "single_word": false,
       "lstrip": false,
       "special": true
     },
     {
+      "id": 1,
       "content": "<bos>",
       "single_word": false,
       "lstrip": false,
       "special": true
     },
     {
+      "id": 2,
       "content": "<eos>",
       "single_word": false,
       "lstrip": false,
   "model": {
     "type": "WordLevel",
     "vocab": {
+      ",": 0,
+      "the": 1,
+      ".": 2,
+      "of": 3,
+      "and": 4,
+      "to": 5,
+      "a": 6,
+      "in": 7,
+      "\"": 8,
+      "that": 9,
+      "was": 10,
+      "is": 11,
+      "for": 12,
+      "on": 13,
+      "with": 14,
+      "as": 15,
+      "it": 16,
+      "he": 17,
+      "his": 18,
+      "at": 19,
+      "by": 20,
+      "i": 21,
+      ";": 22,
+      "be": 23,
+      "from": 24,
+      "<bos>": 25,
+      "<eos>": 26,
+      "not": 27,
+      ")": 28,
+      "but": 29,
+      "(": 30,
+      "this": 31,
+      "had": 32,
+      "which": 33,
+      "have": 34,
+      "are": 35,
+      ":": 36,
+      "they": 37,
+      "an": 38,
+      "or": 39,
+      "were": 40,
+      "her": 41,
+      "said": 42,
+      "their": 43,
+      "has": 44,
+      "we": 45,
+      "all": 46,
+      "who": 47,
+      "one": 48,
+      "she": 49,
+      "you": 50,
+      "“": 51,
+      "”": 52,
+      "been": 53,
+      "will": 54,
+      "more": 55,
+      "him": 56,
+      "when": 57,
+      "so": 58,
+      "its": 59,
+      "would": 60,
+      "there": 61,
+      "no": 62,
+      "after": 63,
+      "my": 64,
+      "if": 65,
+      "out": 66,
+      "also": 67,
+      "about": 68,
+      "them": 69,
+      "up": 70,
+      "into": 71,
+      "two": 72,
+      "what": 73,
+      "new": 74,
+      "first": 75,
+      "some": 76,
+      "can": 77,
+      "time": 78,
+      "than": 79,
+      "other": 80,
+      "our": 81,
+      "me": 82,
+      "only": 83,
+      "over": 84,
+      "could": 85,
+      "may": 86,
+      "?": 87,
+      "any": 88,
+      "/": 89,
+      "then": 90,
+      "people": 91,
+      "like": 92,
+      "now": 93,
+      "such": 94,
+      "do": 95,
+      "us": 96,
+      "very": 97,
+      "made": 98,
+      "these": 99
     },
     "unk_token": "<unk>"
   }