Upload mgpt2 tokenizer

added_tokens.json

@@ -1,3 +1,3 @@
 {
-  "<|endoftext|>": 50252
+  "<|endoftext|>": 50256
 }

evaluation.json

@@ -1,5 +1,5 @@
 {
-  "text": "tokenizer/artifacts/heldout_eval_50k.txt",
+  "text": "tokenizer/artifacts/heldout_eval.txt",
   "limit": 10000,
   "overall": [
     {
@@ -32,14 +32,14 @@
       "name": "mgpt2_RegexTokenizer_candidate (tokenizer/artifacts/mgpt2.model)",
       "total_chars": 43290048,
       "total_bytes": 43442607,
-      "total_tokens": 8971173,
-      "tokens_per_1k_chars": 207.23407375293277,
-      "tokens_per_1k_bytes": 206.50632223798172,
-      "bytes_per_token": 4.842466754347509,
-      "chars_per_token": 4.825461285831853,
-      "p50_tokens_per_line": 510,
-      "p95_tokens_per_line": 2678,
-      "p95_tokens_per_1k_bytes_per_line": 261.3065326633166
+      "total_tokens": 8960195,
+      "tokens_per_1k_chars": 206.98048198052356,
+      "tokens_per_1k_bytes": 206.25362101312197,
+      "bytes_per_token": 4.848399727907707,
+      "chars_per_token": 4.831373424350698,
+      "p50_tokens_per_line": 509,
+      "p95_tokens_per_line": 2677,
+      "p95_tokens_per_1k_bytes_per_line": 260.9841827768014
     }
   ],
   "by_bucket": {
@@ -74,14 +74,14 @@
         "name": "mgpt2_RegexTokenizer_candidate (tokenizer/artifacts/mgpt2.model)",
         "total_chars": 42393232,
         "total_bytes": 42542626,
-        "total_tokens": 8795657,
-        "tokens_per_1k_chars": 207.47785872990292,
-        "tokens_per_1k_bytes": 206.74927307026135,
-        "bytes_per_token": 4.836776377250727,
-        "chars_per_token": 4.8197914038712515,
-        "p50_tokens_per_line": 508,
-        "p95_tokens_per_line": 2643,
-        "p95_tokens_per_1k_bytes_per_line": 261.53846153846155
+        "total_tokens": 8786409,
+        "tokens_per_1k_chars": 207.25971070099115,
+        "tokens_per_1k_bytes": 206.5318910967085,
+        "bytes_per_token": 4.841867252025258,
+        "chars_per_token": 4.82486440137262,
+        "p50_tokens_per_line": 507,
+        "p95_tokens_per_line": 2644,
+        "p95_tokens_per_1k_bytes_per_line": 261.04640414124606
       }
     ],
     "mixed": [
@@ -115,14 +115,14 @@
         "name": "mgpt2_RegexTokenizer_candidate (tokenizer/artifacts/mgpt2.model)",
         "total_chars": 896816,
         "total_bytes": 899981,
-        "total_tokens": 175516,
-        "tokens_per_1k_chars": 195.71015682146617,
-        "tokens_per_1k_bytes": 195.02189490667024,
-        "bytes_per_token": 5.12762938991317,
-        "chars_per_token": 5.109596845871601,
-        "p50_tokens_per_line": 930,
-        "p95_tokens_per_line": 6963,
-        "p95_tokens_per_1k_bytes_per_line": 256.91514299109235
+        "total_tokens": 173786,
+        "tokens_per_1k_chars": 193.78111006048064,
+        "tokens_per_1k_bytes": 193.0996321033444,
+        "bytes_per_token": 5.178673771189854,
+        "chars_per_token": 5.160461717284476,
+        "p50_tokens_per_line": 924,
+        "p95_tokens_per_line": 6938,
+        "p95_tokens_per_1k_bytes_per_line": 248.08184143222508
       }
     ]
   }

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a54fe6d4082efbb4fdbcf662349226590b07ed6f2ff85dedf29d4585881d1e8
-size 460673
+oid sha256:b33c900b86a6fd548544ce693be7800b55cfd81576f0b2c3a9b1a91836c069ec
+size 459544

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "added_tokens_decoder": {
-    "50252": {
+    "50256": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,