divilian
/

lilchatbot-tokenizer

divilian commited on Feb 3

Commit

cdddbaa

verified ·

1 Parent(s): f4817aa

Upload tokenizer.json with huggingface_hub

Files changed (1) hide show

tokenizer.json CHANGED Viewed

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "<unk",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -93,7 +93,7 @@
   "model": {
     "type": "WordLevel",
     "vocab": {
-      "<unk": 0,
       "<pad>": 1,
       ",": 4,
       "the": 5,

   "added_tokens": [
     {
       "id": 0,
+      "content": "<unk>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
   "model": {
     "type": "WordLevel",
     "vocab": {
+      "<unk>": 0,
       "<pad>": 1,
       ",": 4,
       "the": 5,