Upload 12 files

src/tokenizers/bpe_v15k_f2/special_tokens_map.json

@@ -0,0 +1,3 @@
+{
+  "unk_token": "<UNK>"
+}

src/tokenizers/bpe_v15k_f2/tokenizer_config.json

@@ -0,0 +1,41 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<UNK>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<NUM>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<URL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "unk_token": "<UNK>",
+  "cls_token": null,
+  "sep_token": null,
+  "mask_token": null,
+  "model_max_length": 512
+}

src/tokenizers/bpe_v8k_f2/special_tokens_map.json

@@ -0,0 +1,3 @@
+{
+  "unk_token": "<UNK>"
+}

src/tokenizers/bpe_v8k_f2/tokenizer_config.json

@@ -0,0 +1,7 @@
+{
+  "unk_token": "<UNK>",
+  "cls_token": null,
+  "sep_token": null,
+  "mask_token": null,
+  "model_max_length": 512
+}

src/tokenizers/bpe_v8k_f3/special_tokens_map.json

@@ -0,0 +1,3 @@
+{
+  "unk_token": "<UNK>"
+}

src/tokenizers/bpe_v8k_f3/tokenizer_config.json

@@ -0,0 +1,7 @@
+{
+  "unk_token": "<UNK>",
+  "cls_token": null,
+  "sep_token": null,
+  "mask_token": null,
+  "model_max_length": 512
+}

src/tokenizers/bpe_v8k_f4/special_tokens_map.json

@@ -0,0 +1,3 @@
+{
+  "unk_token": "<UNK>"
+}

src/tokenizers/bpe_v8k_f4/tokenizer_config.json

@@ -0,0 +1,7 @@
+{
+  "unk_token": "<UNK>",
+  "cls_token": null,
+  "sep_token": null,
+  "mask_token": null,
+  "model_max_length": 512
+}