EleutherAI
/

polyglot-ko-3.8b

Text Generation

text-generation-inference

Model card Files Files and versions

hyunwoongko commited on Oct 8, 2022

Commit

7a6ce4c

·

1 Parent(s): b4eab72

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +5 -5

tokenizer.json CHANGED Viewed

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "<pad>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "<eos>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -107,8 +107,8 @@
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "vocab": {
-      "<pad>": 0,
-      "<eos>": 1,
       "<|endoftext|>": 2,
       "<|sep|>": 3,
       "!": 4,
@@ -59851,4 +59851,4 @@
       "ìºĲë¡¤ëĿ¼ ìĿ´ëĤĺ"
     ]
   }
-}

   "added_tokens": [
     {
       "id": 0,
+      "content": "<|unused0|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
     },
     {
       "id": 1,
+      "content": "<|unused1|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "vocab": {
+      "<|unused0|>": 0,
+      "<|unused1|>": 1,
       "<|endoftext|>": 2,
       "<|sep|>": 3,
       "!": 4,
       "ìºĲë¡¤ëĿ¼ ìĿ´ëĤĺ"
     ]
   }
+}