Joycean0301
/

test-tokenizer

Model card Files Files and versions

Joycean0301 commited on Oct 18, 2024

Commit

ff189f8

·

verified ·

1 Parent(s): e741b31

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +2 -2

tokenizer_config.json CHANGED Viewed

@@ -315,7 +315,7 @@
       "single_word": false,
       "special": false
     },
-    "50296": {
       "content": "<|beginoftext|>",
       "lstrip": false,
       "normalized": true,
@@ -323,7 +323,7 @@
       "single_word": false,
       "special": false
     },
-    "50297": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,

       "single_word": false,
       "special": false
     },
+    "50300": {
       "content": "<|beginoftext|>",
       "lstrip": false,
       "normalized": true,
       "single_word": false,
       "special": false
     },
+    "50400": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,