Tokenizer update - 20251212_0722

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
-size 11422654
+oid sha256:2ea7d9a2224ffb61ccf08a6786eed3428abf949b1e9cea729cbddc4c52c02b18
+size 11422752

tokenizer_config.json

@@ -231,10 +231,14 @@
   "eos_token": "<|im_end|>",
   "errors": "replace",
   "extra_special_tokens": {},
+  "max_length": 128,
   "model_max_length": 1010000,
   "pad_token": "<|endoftext|>",
   "padding_side": "left",
   "split_special_tokens": false,
+  "stride": 0,
   "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": null
 }