Make tokenizer metadata compatible with Transformers 4 and 5

cause/tokenizer/tokenizer_config.json

@@ -5,7 +5,14 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": [
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -19,12 +26,5 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>"
-  ],
-  "is_local": true,
-  "local_files_only": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  ]
 }

checksums.json

@@ -1,48 +1,48 @@
 {
   "general/head.safetensors": "34387f1e00d7b105d2322383a7f1d02bbdfef5aa09569b406f050abb6247758e",
-  "general/tokenizer/tokenizer_config.json": "5f007d04324afa93876e2a484190e868136c69dfe0e1a29a1063eae583df9d32",
+  "general/tokenizer/tokenizer_config.json": "e43d3f72ef651b06e636724ea083f3f15cd8f47643f71019c664d474adabfb93",
   "general/tokenizer/tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
   "general/tokenizer/chat_template.jinja": "87a2728cb8dc9fe424d624542f6060ec05a1d285ebbec578bb078900e33396b5",
   "general/adapter/adapter_model.safetensors": "5fd2586ab619fab7044d8d37c753d5ef582a3a4f0eed9f3ea76611d72ffe09f9",
   "general/adapter/adapter_config.json": "26f6f2eaa4825b3e581ab0f3c8433c9618d5fa9ba6c1edc5ebe1cd8a4c6bb732",
   "intent/head.safetensors": "a335ef44d661ace9c917ad5e7cde687f05bb8dcf5ec6da43b95bea6e14758cb8",
-  "intent/tokenizer/tokenizer_config.json": "5f007d04324afa93876e2a484190e868136c69dfe0e1a29a1063eae583df9d32",
+  "intent/tokenizer/tokenizer_config.json": "e43d3f72ef651b06e636724ea083f3f15cd8f47643f71019c664d474adabfb93",
   "intent/tokenizer/tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
   "intent/tokenizer/chat_template.jinja": "87a2728cb8dc9fe424d624542f6060ec05a1d285ebbec578bb078900e33396b5",
   "intent/adapter/adapter_model.safetensors": "7657e89382dc6a93176169cb652106e5a64debbb690087609470cb56b2aab97f",
   "intent/adapter/adapter_config.json": "e9e31b572e42208c6145aac7d3f21cbe11f666bff10c83ed0562068069fc6124",
   "sentiment/head.safetensors": "b4e29533d2e26a2744ce94cdb7203aad410d727a556fb105d87bc321529e8dc1",
-  "sentiment/tokenizer/tokenizer_config.json": "5f007d04324afa93876e2a484190e868136c69dfe0e1a29a1063eae583df9d32",
+  "sentiment/tokenizer/tokenizer_config.json": "e43d3f72ef651b06e636724ea083f3f15cd8f47643f71019c664d474adabfb93",
   "sentiment/tokenizer/tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
   "sentiment/tokenizer/chat_template.jinja": "87a2728cb8dc9fe424d624542f6060ec05a1d285ebbec578bb078900e33396b5",
   "sentiment/adapter/adapter_model.safetensors": "87f22f6edc58dd0c5c62fc23b75d0fce13a056549010f87743d3daa469400c18",
   "sentiment/adapter/adapter_config.json": "26f6f2eaa4825b3e581ab0f3c8433c9618d5fa9ba6c1edc5ebe1cd8a4c6bb732",
   "triage/head.safetensors": "c156d71a993881378c94e09f0a37c0a57ec8d61588145f6c1ebc0f425098bde3",
-  "triage/tokenizer/tokenizer_config.json": "5f007d04324afa93876e2a484190e868136c69dfe0e1a29a1063eae583df9d32",
+  "triage/tokenizer/tokenizer_config.json": "e43d3f72ef651b06e636724ea083f3f15cd8f47643f71019c664d474adabfb93",
   "triage/tokenizer/tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
   "triage/tokenizer/chat_template.jinja": "87a2728cb8dc9fe424d624542f6060ec05a1d285ebbec578bb078900e33396b5",
   "triage/adapter/adapter_model.safetensors": "aff6f85e1421a1e586b677112df5d7e4e5c6ae624aa8117dcf49ab60d41c7121",
   "triage/adapter/adapter_config.json": "26f6f2eaa4825b3e581ab0f3c8433c9618d5fa9ba6c1edc5ebe1cd8a4c6bb732",
   "cause/head.safetensors": "d180c8ec0249df0d4c4e6d7a541e4e8cfbae570d871a18de42fac3143ee33107",
-  "cause/tokenizer/tokenizer_config.json": "5f007d04324afa93876e2a484190e868136c69dfe0e1a29a1063eae583df9d32",
+  "cause/tokenizer/tokenizer_config.json": "e43d3f72ef651b06e636724ea083f3f15cd8f47643f71019c664d474adabfb93",
   "cause/tokenizer/tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
   "cause/tokenizer/chat_template.jinja": "87a2728cb8dc9fe424d624542f6060ec05a1d285ebbec578bb078900e33396b5",
   "cause/adapter/adapter_model.safetensors": "f7d7613b1af5a5fcf8c3ad0652f455c883b933229d5e2cd10c5fb1894a9a24f9",
   "cause/adapter/adapter_config.json": "26f6f2eaa4825b3e581ab0f3c8433c9618d5fa9ba6c1edc5ebe1cd8a4c6bb732",
   "codify/head.safetensors": "bd2011c3ab9159d89ef69890119540c87a3091e38ef88dccd2f59f6daa04d559",
-  "codify/tokenizer/tokenizer_config.json": "5f007d04324afa93876e2a484190e868136c69dfe0e1a29a1063eae583df9d32",
+  "codify/tokenizer/tokenizer_config.json": "e43d3f72ef651b06e636724ea083f3f15cd8f47643f71019c664d474adabfb93",
   "codify/tokenizer/tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
   "codify/tokenizer/chat_template.jinja": "87a2728cb8dc9fe424d624542f6060ec05a1d285ebbec578bb078900e33396b5",
   "codify/adapter/adapter_model.safetensors": "e7bed2b463ac651a071baa897b55276153a47f08a77286ced9c3fbe309a2da57",
   "codify/adapter/adapter_config.json": "26f6f2eaa4825b3e581ab0f3c8433c9618d5fa9ba6c1edc5ebe1cd8a4c6bb732",
   "complaints/head.safetensors": "a9c8e7ac2ca0342bd0cf9a7c8409a3b84c1c782b934327d4ea85ba693781e8f7",
-  "complaints/tokenizer/tokenizer_config.json": "5f007d04324afa93876e2a484190e868136c69dfe0e1a29a1063eae583df9d32",
+  "complaints/tokenizer/tokenizer_config.json": "e43d3f72ef651b06e636724ea083f3f15cd8f47643f71019c664d474adabfb93",
   "complaints/tokenizer/tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
   "complaints/tokenizer/chat_template.jinja": "87a2728cb8dc9fe424d624542f6060ec05a1d285ebbec578bb078900e33396b5",
   "complaints/adapter/adapter_model.safetensors": "67b6ff8f6bea1cbf631e9f76dbc0a5f639e2304579e7447aa42d047d1f1f092c",
   "complaints/adapter/adapter_config.json": "26f6f2eaa4825b3e581ab0f3c8433c9618d5fa9ba6c1edc5ebe1cd8a4c6bb732",
   "stars/head.safetensors": "83137bbf086729f8c93cc0bdb9f1ad3e83d2435747aca1780c4b67cc4057bf1f",
-  "stars/tokenizer/tokenizer_config.json": "5f007d04324afa93876e2a484190e868136c69dfe0e1a29a1063eae583df9d32",
+  "stars/tokenizer/tokenizer_config.json": "e43d3f72ef651b06e636724ea083f3f15cd8f47643f71019c664d474adabfb93",
   "stars/tokenizer/tokenizer.json": "be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506",
   "stars/tokenizer/chat_template.jinja": "87a2728cb8dc9fe424d624542f6060ec05a1d285ebbec578bb078900e33396b5",
   "stars/adapter/adapter_model.safetensors": "46eb9bca05dd7690d3feea5c697b91573da5f9765595daaf75929c5324931fdb",

codify/tokenizer/tokenizer_config.json

@@ -5,7 +5,14 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": [
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -19,12 +26,5 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>"
-  ],
-  "is_local": true,
-  "local_files_only": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  ]
 }

complaints/tokenizer/tokenizer_config.json

@@ -5,7 +5,14 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": [
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -19,12 +26,5 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>"
-  ],
-  "is_local": true,
-  "local_files_only": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  ]
 }

general/tokenizer/tokenizer_config.json

@@ -5,7 +5,14 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": [
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -19,12 +26,5 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>"
-  ],
-  "is_local": true,
-  "local_files_only": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  ]
 }

intent/tokenizer/tokenizer_config.json

@@ -5,7 +5,14 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": [
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -19,12 +26,5 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>"
-  ],
-  "is_local": true,
-  "local_files_only": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  ]
 }

sentiment/tokenizer/tokenizer_config.json

@@ -5,7 +5,14 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": [
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -19,12 +26,5 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>"
-  ],
-  "is_local": true,
-  "local_files_only": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  ]
 }

stars/tokenizer/tokenizer_config.json

@@ -5,7 +5,14 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": [
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -19,12 +26,5 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>"
-  ],
-  "is_local": true,
-  "local_files_only": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  ]
 }

triage/tokenizer/tokenizer_config.json

@@ -5,7 +5,14 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": [
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -19,12 +26,5 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>"
-  ],
-  "is_local": true,
-  "local_files_only": false,
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  ]
 }