Upload tokenizer for fine-tuned RAFT generator model

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
-size 11422654
+oid sha256:2c7cc7e0c7e701d98da66c9d81be86c724b491c8b8b175caddcd3538d298fcf1
+size 11422934

tokenizer_config.json

@@ -228,8 +228,15 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "extra_special_tokens": {},
+  "max_length": 1500,
   "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
   "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "stride": 0,
   "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<|endoftext|>"
 }