ikkiren
/

research_tokenizer

Model card Files Files and versions

ikkiren commited on Apr 21, 2025

Commit

0bf9b2d

·

verified ·

1 Parent(s): f6f4192

Upload tokenizer

Files changed (2) hide show

tokenizer.json +2 -2
tokenizer_config.json +1 -1

tokenizer.json CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
-size 17209920

 version https://git-lfs.github.com/spec/v1
+oid sha256:d9bec79b7e24f121d0d1acb81a8d5b253784c1348bf15e003423143b65b26821
+size 14067110

tokenizer_config.json CHANGED Viewed

@@ -2059,5 +2059,5 @@
     "attention_mask"
   ],
   "model_max_length": 131072,
-  "tokenizer_class": "PreTrainedTokenizerFast"
 }

     "attention_mask"
   ],
   "model_max_length": 131072,
+  "tokenizer_class": "PreTrainedTokenizer"
 }