Upload tokenizer
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +21 -0
tokenizer.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:12bac982b793c44b03d52a250a9f0d0b666813da566b910c24a6da0695fd11e6
|
| 3 |
+
size 32170070
|
tokenizer_config.json
CHANGED
|
@@ -17,8 +17,29 @@
|
|
| 17 |
"<|video|>"
|
| 18 |
],
|
| 19 |
"image_token": "<|image|>",
|
|
|
|
| 20 |
"mask_token": "<mask>",
|
| 21 |
"model_max_length": 1000000000000000019884624838656,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
"pad_token": "<pad>",
|
| 23 |
"padding_side": "left",
|
| 24 |
"processor_class": "Gemma4Processor",
|
|
|
|
| 17 |
"<|video|>"
|
| 18 |
],
|
| 19 |
"image_token": "<|image|>",
|
| 20 |
+
"is_local": false,
|
| 21 |
"mask_token": "<mask>",
|
| 22 |
"model_max_length": 1000000000000000019884624838656,
|
| 23 |
+
"model_specific_special_tokens": {
|
| 24 |
+
"audio_token": "<|audio|>",
|
| 25 |
+
"boa_token": "<|audio>",
|
| 26 |
+
"boi_token": "<|image>",
|
| 27 |
+
"eoa_token": "<audio|>",
|
| 28 |
+
"eoc_token": "<channel|>",
|
| 29 |
+
"eoi_token": "<image|>",
|
| 30 |
+
"eot_token": "<turn|>",
|
| 31 |
+
"escape_token": "<|\"|>",
|
| 32 |
+
"etc_token": "<tool_call|>",
|
| 33 |
+
"etd_token": "<tool|>",
|
| 34 |
+
"etr_token": "<tool_response|>",
|
| 35 |
+
"image_token": "<|image|>",
|
| 36 |
+
"soc_token": "<|channel>",
|
| 37 |
+
"sot_token": "<|turn>",
|
| 38 |
+
"stc_token": "<|tool_call>",
|
| 39 |
+
"std_token": "<|tool>",
|
| 40 |
+
"str_token": "<|tool_response>",
|
| 41 |
+
"think_token": "<|think|>"
|
| 42 |
+
},
|
| 43 |
"pad_token": "<pad>",
|
| 44 |
"padding_side": "left",
|
| 45 |
"processor_class": "Gemma4Processor",
|