add tokenizer

.gitattributes

@@ -29,3 +29,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+vocab.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|endoftext|>": 27671
+}

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<|endoftext|>",
+  "cls_token": "[CLS]",
+  "eos_token": "<|endoftext|>",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4028d2189159ae52d62a38d4b0cd2d572e497c904b0ad364f228c6cdffc2197
+size 635247

tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "mask_token": "[MASK]",
+  "name_or_path": "bigmorning/distilgpt_new2_0060",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_token": [
+    "[PAD]",
+    "[UNK]",
+    "[CLS]",
+    "[SEP]",
+    "[MASK]"
+  ],
+  "special_tokens_map_file": "/root/.cache/huggingface/transformers/f9dd91ec01fc25f5ae3f4dd46e36ac7fc9a7c2e42c82a107dca01bc6b97764ec.7da70648c6cb9951e284c9685f9ba7ae083dd59ed1d6d84bdfc0584a4ea94b6d",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "[UNK]"
+}

vocab.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1389526b69c5a65ba6c9d3b7fe2ad76192337ca89162d6f448e80994e5889fe
+size 412436