add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Jul22_19-08-08_600ca18b0e9a/1626980915.4764636/events.out.tfevents.1626980915.600ca18b0e9a.62.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8f063f555d7b1d67805defb5b5ee98684331e3420476336322b0ecf64c5143f
+size 4174

runs/Jul22_19-08-08_600ca18b0e9a/events.out.tfevents.1626980915.600ca18b0e9a.62.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a58b82e3f6ec157e641a9670d449ab52241da68991a63c1ac79732e95689b91
+size 220162

runs/Jul22_19-08-08_600ca18b0e9a/events.out.tfevents.1626981013.600ca18b0e9a.62.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:105979b1e70d3e1e122e3a1991825efd18eab3ac5f36dbaf118792b31fc5c5f4
+size 311

runs/Jul22_19-10-50_600ca18b0e9a/1626981056.06171/events.out.tfevents.1626981056.600ca18b0e9a.62.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd5ab67c9a6b326c3a61aaa7e2df176b7e17d3dd33de59181aada3adae2dc029
+size 4174

runs/Jul22_19-10-50_600ca18b0e9a/events.out.tfevents.1626981056.600ca18b0e9a.62.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e650c768fade8789186a77839b959772635c09a1ffd3458f7828df4788f27831
+size 89800

runs/Jul22_19-11-45_600ca18b0e9a/1626981106.562775/events.out.tfevents.1626981106.600ca18b0e9a.62.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c48706ea19bc5663dfbbc329b5d303846f39c7ddcae60215a8a10dbe233618e7
+size 4174

runs/Jul22_19-11-45_600ca18b0e9a/1626981143.6726632/events.out.tfevents.1626981143.600ca18b0e9a.62.8

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bd6d0a7279cebea7d17e9bad7dfd98f71a006f4f42c8096d7a132c9b93e8e24
+size 4174

runs/Jul22_19-11-45_600ca18b0e9a/events.out.tfevents.1626981106.600ca18b0e9a.62.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1e95302cb1972b85e948c238745f265d40dc5366c6c00c60c9fa416f0707926
+size 23210

runs/Jul22_19-11-45_600ca18b0e9a/events.out.tfevents.1626981143.600ca18b0e9a.62.7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59c54327435c779d08306081ccaf5e88380782ff03db3753f6ab869ab6d2f2df
+size 3960

runs/Jul22_19-12-38_600ca18b0e9a/1626981160.0041342/events.out.tfevents.1626981160.600ca18b0e9a.62.10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee2fec38da7acfcfbecdc4ac8745bb0032e75601f61c0e9051793565dba2af66
+size 4174

runs/Jul22_19-12-38_600ca18b0e9a/1626981428.1395035/events.out.tfevents.1626981428.600ca18b0e9a.62.12

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b157ab2d676060e715b7f6de59cf40d9c1612c36c08c3a05903e5ff9af60532a
+size 4174

runs/Jul22_19-12-38_600ca18b0e9a/events.out.tfevents.1626981160.600ca18b0e9a.62.9

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95e339975d5d0c7d96468ec59a6af36eaefda3c7c34cd69729506b9ce999d7b1
+size 3960

runs/Jul22_19-12-38_600ca18b0e9a/events.out.tfevents.1626981428.600ca18b0e9a.62.11

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9081f487789766b2b535610142075fa788ebe69b8d29abc7e7e8ac80bc1c534
+size 436393

runs/Jul22_19-12-38_600ca18b0e9a/events.out.tfevents.1626981635.600ca18b0e9a.62.13

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64dd8928b010fffc4449176712ed909d5679bc8dbfae8992e27aa7ada0dc3997
+size 311

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "facebook/bart-base", "tokenizer_class": "BartTokenizer"}