ArthurZ
/

test-push-tok

Model card Files Files and versions

Upload tokenizer

#4

by ArthurZ HF Staff - opened Oct 9, 2023

base: refs/heads/main

←

from: refs/pr/4

Discussion Files changed

Files changed (3) hide show

special_tokens_map.json +1 -1
tokenizer.json +2 -2
tokenizer_config.json +1 -1

special_tokens_map.json CHANGED Viewed

@@ -227,7 +227,7 @@
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
-    "normalized": true,
     "rstrip": false,
     "single_word": false
   },

   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
+    "normalized": false,
     "rstrip": false,
     "single_word": false
   },

tokenizer.json CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:610501fe8857739dbb451ab69a0a795cb87dadcf8873d7e2227764d165e72e72
-size 17331379

 version https://git-lfs.github.com/spec/v1
+oid sha256:a9633f7d33a10432bf06e8865c3c7d4e4798ed432be7bbc7a4a29051a7f5e594
+size 17331380

tokenizer_config.json CHANGED Viewed

@@ -1651,7 +1651,7 @@
     "256203": {
       "content": "<mask>",
       "lstrip": true,
-      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true

     "256203": {
       "content": "<mask>",
       "lstrip": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true