Upload 5 files

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "mask_token": "…",
+  "unk_token": "<unk>"
+}

token_fraction_describe.txt

@@ -0,0 +1,16 @@
+              token          text    token/text
+count  19980.000000  19980.000000  19980.000000
+mean      92.362813    181.553804      0.511898
+std      244.779002    374.992707      0.086487
+min        8.000000     41.000000      0.181818
+10%       24.000000     46.000000      0.415385
+25%       34.000000     64.000000      0.456522
+33%       43.000000     85.000000      0.472603
+50%       69.000000    139.000000      0.504119
+67%      103.000000    205.000000      0.538462
+75%      124.000000    245.000000      0.559595
+80%      140.000000    277.000000      0.574468
+90%      188.000000    371.000000      0.619768
+95%      236.000000    461.000000      0.665354
+99%      343.210000    666.000000      0.760399
+max    33092.000000  49156.000000      1.268293

tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "…",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "mask_token": "…",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}