Add all tokenizer, processor, and prompt configs
Browse files- .gitattributes +35 -0
- config.json +165 -0
- generation_config.json +14 -0
- processor_config.json +62 -0
- prompt_format.json +8 -0
- tokenizer.json +3 -0
- tokenizer_config.json +55 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,38 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
gguf/humanizer-gemma-4-e4b-bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
gguf/humanizer-gemma-4-e4b-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
gguf/humanizer-gemma-4-e4b-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
gguf/humanizer-gemma-4-e4b-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
gguf/humanizer-gemma-4-e4b-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
lite/humanizer-lite-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
lite/humanizer-lite-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
lite/humanizer-lite-bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
humanizer-12b-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
humanizer-12b-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
assets/results-detector-en.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
assets/training-zh.png filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
assets/app-showcase-zh.png filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
assets/banner-zh.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
assets/compare-en-email.png filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
assets/compare-zh-zhihu.png filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
assets/training-en.png filter=lfs diff=lfs merge=lfs -text
|
| 54 |
+
assets/results-detector-zh.png filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
assets/banner-en.png filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
assets/app-showcase-en.png filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
assets/compare-en-reddit.png filter=lfs diff=lfs merge=lfs -text
|
| 58 |
+
assets/results-fidelity-en.png filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
assets/compare-zh-email.png filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
assets/app-en.png filter=lfs diff=lfs merge=lfs -text
|
| 61 |
+
assets/app-zh.png filter=lfs diff=lfs merge=lfs -text
|
| 62 |
+
assets/results-fidelity-zh.png filter=lfs diff=lfs merge=lfs -text
|
| 63 |
+
humanizer-12b-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
assets/compare-en-forum.png filter=lfs diff=lfs merge=lfs -text
|
| 65 |
+
humanizer-12b-bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
| 66 |
+
humanizer-12b-IQ2_XS-QAT.gguf filter=lfs diff=lfs merge=lfs -text
|
| 67 |
+
humanizer-12b-Q3-QAT.gguf filter=lfs diff=lfs merge=lfs -text
|
| 68 |
+
assets/quant-top1-en.png filter=lfs diff=lfs merge=lfs -text
|
| 69 |
+
assets/eval-en.png filter=lfs diff=lfs merge=lfs -text
|
| 70 |
+
assets/eval-zh.png filter=lfs diff=lfs merge=lfs -text
|
config.json
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Gemma4UnifiedForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"audio_config": {
|
| 6 |
+
"_name_or_path": "",
|
| 7 |
+
"architectures": null,
|
| 8 |
+
"audio_embed_dim": 640,
|
| 9 |
+
"chunk_size_feed_forward": 0,
|
| 10 |
+
"dtype": "bfloat16",
|
| 11 |
+
"id2label": {
|
| 12 |
+
"0": "LABEL_0",
|
| 13 |
+
"1": "LABEL_1"
|
| 14 |
+
},
|
| 15 |
+
"initializer_range": 0.02,
|
| 16 |
+
"is_encoder_decoder": false,
|
| 17 |
+
"label2id": {
|
| 18 |
+
"LABEL_0": 0,
|
| 19 |
+
"LABEL_1": 1
|
| 20 |
+
},
|
| 21 |
+
"model_type": "gemma4_unified_audio",
|
| 22 |
+
"output_attentions": false,
|
| 23 |
+
"output_hidden_states": false,
|
| 24 |
+
"problem_type": null,
|
| 25 |
+
"return_dict": true,
|
| 26 |
+
"rms_norm_eps": 1e-06
|
| 27 |
+
},
|
| 28 |
+
"audio_token_id": 258881,
|
| 29 |
+
"boa_token_id": 256000,
|
| 30 |
+
"boi_token_id": 255999,
|
| 31 |
+
"dtype": "bfloat16",
|
| 32 |
+
"eoa_token_index": 258883,
|
| 33 |
+
"eoi_token_id": 258882,
|
| 34 |
+
"image_token_id": 258880,
|
| 35 |
+
"initializer_range": 0.02,
|
| 36 |
+
"model_type": "gemma4_unified",
|
| 37 |
+
"text_config": {
|
| 38 |
+
"attention_bias": false,
|
| 39 |
+
"attention_dropout": 0.0,
|
| 40 |
+
"attention_k_eq_v": true,
|
| 41 |
+
"bos_token_id": 2,
|
| 42 |
+
"dtype": "bfloat16",
|
| 43 |
+
"enable_moe_block": false,
|
| 44 |
+
"eos_token_id": 1,
|
| 45 |
+
"final_logit_softcapping": 30.0,
|
| 46 |
+
"global_head_dim": 512,
|
| 47 |
+
"head_dim": 256,
|
| 48 |
+
"hidden_activation": "gelu_pytorch_tanh",
|
| 49 |
+
"hidden_size": 3840,
|
| 50 |
+
"hidden_size_per_layer_input": 0,
|
| 51 |
+
"initializer_range": 0.02,
|
| 52 |
+
"intermediate_size": 15360,
|
| 53 |
+
"layer_types": [
|
| 54 |
+
"sliding_attention",
|
| 55 |
+
"sliding_attention",
|
| 56 |
+
"sliding_attention",
|
| 57 |
+
"sliding_attention",
|
| 58 |
+
"sliding_attention",
|
| 59 |
+
"full_attention",
|
| 60 |
+
"sliding_attention",
|
| 61 |
+
"sliding_attention",
|
| 62 |
+
"sliding_attention",
|
| 63 |
+
"sliding_attention",
|
| 64 |
+
"sliding_attention",
|
| 65 |
+
"full_attention",
|
| 66 |
+
"sliding_attention",
|
| 67 |
+
"sliding_attention",
|
| 68 |
+
"sliding_attention",
|
| 69 |
+
"sliding_attention",
|
| 70 |
+
"sliding_attention",
|
| 71 |
+
"full_attention",
|
| 72 |
+
"sliding_attention",
|
| 73 |
+
"sliding_attention",
|
| 74 |
+
"sliding_attention",
|
| 75 |
+
"sliding_attention",
|
| 76 |
+
"sliding_attention",
|
| 77 |
+
"full_attention",
|
| 78 |
+
"sliding_attention",
|
| 79 |
+
"sliding_attention",
|
| 80 |
+
"sliding_attention",
|
| 81 |
+
"sliding_attention",
|
| 82 |
+
"sliding_attention",
|
| 83 |
+
"full_attention",
|
| 84 |
+
"sliding_attention",
|
| 85 |
+
"sliding_attention",
|
| 86 |
+
"sliding_attention",
|
| 87 |
+
"sliding_attention",
|
| 88 |
+
"sliding_attention",
|
| 89 |
+
"full_attention",
|
| 90 |
+
"sliding_attention",
|
| 91 |
+
"sliding_attention",
|
| 92 |
+
"sliding_attention",
|
| 93 |
+
"sliding_attention",
|
| 94 |
+
"sliding_attention",
|
| 95 |
+
"full_attention",
|
| 96 |
+
"sliding_attention",
|
| 97 |
+
"sliding_attention",
|
| 98 |
+
"sliding_attention",
|
| 99 |
+
"sliding_attention",
|
| 100 |
+
"sliding_attention",
|
| 101 |
+
"full_attention"
|
| 102 |
+
],
|
| 103 |
+
"max_position_embeddings": 262144,
|
| 104 |
+
"model_type": "gemma4_unified_text",
|
| 105 |
+
"moe_intermediate_size": null,
|
| 106 |
+
"num_attention_heads": 16,
|
| 107 |
+
"num_experts": null,
|
| 108 |
+
"num_global_key_value_heads": 1,
|
| 109 |
+
"num_hidden_layers": 48,
|
| 110 |
+
"num_key_value_heads": 8,
|
| 111 |
+
"num_kv_shared_layers": 0,
|
| 112 |
+
"pad_token_id": 0,
|
| 113 |
+
"rms_norm_eps": 1e-06,
|
| 114 |
+
"rope_parameters": {
|
| 115 |
+
"full_attention": {
|
| 116 |
+
"partial_rotary_factor": 0.25,
|
| 117 |
+
"rope_theta": 1000000.0,
|
| 118 |
+
"rope_type": "proportional"
|
| 119 |
+
},
|
| 120 |
+
"sliding_attention": {
|
| 121 |
+
"rope_theta": 10000.0,
|
| 122 |
+
"rope_type": "default"
|
| 123 |
+
}
|
| 124 |
+
},
|
| 125 |
+
"sliding_window": 1024,
|
| 126 |
+
"tie_word_embeddings": true,
|
| 127 |
+
"top_k_experts": null,
|
| 128 |
+
"use_bidirectional_attention": "vision",
|
| 129 |
+
"use_cache": true,
|
| 130 |
+
"use_double_wide_mlp": false,
|
| 131 |
+
"vocab_size": 262144,
|
| 132 |
+
"vocab_size_per_layer_input": 262144
|
| 133 |
+
},
|
| 134 |
+
"tie_word_embeddings": true,
|
| 135 |
+
"transformers_version": "5.14.1",
|
| 136 |
+
"video_token_id": 258884,
|
| 137 |
+
"vision_config": {
|
| 138 |
+
"_name_or_path": "",
|
| 139 |
+
"architectures": null,
|
| 140 |
+
"chunk_size_feed_forward": 0,
|
| 141 |
+
"dtype": "bfloat16",
|
| 142 |
+
"id2label": {
|
| 143 |
+
"0": "LABEL_0",
|
| 144 |
+
"1": "LABEL_1"
|
| 145 |
+
},
|
| 146 |
+
"initializer_range": 0.02,
|
| 147 |
+
"is_encoder_decoder": false,
|
| 148 |
+
"label2id": {
|
| 149 |
+
"LABEL_0": 0,
|
| 150 |
+
"LABEL_1": 1
|
| 151 |
+
},
|
| 152 |
+
"mm_embed_dim": 3840,
|
| 153 |
+
"mm_posemb_size": 1120,
|
| 154 |
+
"model_type": "gemma4_unified_vision",
|
| 155 |
+
"num_soft_tokens": 280,
|
| 156 |
+
"output_attentions": false,
|
| 157 |
+
"output_hidden_states": false,
|
| 158 |
+
"output_proj_dims": 3840,
|
| 159 |
+
"patch_size": 16,
|
| 160 |
+
"pooling_kernel_size": 3,
|
| 161 |
+
"problem_type": null,
|
| 162 |
+
"return_dict": true,
|
| 163 |
+
"rms_norm_eps": 1e-06
|
| 164 |
+
}
|
| 165 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 2,
|
| 3 |
+
"do_sample": true,
|
| 4 |
+
"eos_token_id": 1,
|
| 5 |
+
"pad_token_id": 0,
|
| 6 |
+
"suppress_tokens": [
|
| 7 |
+
258883,
|
| 8 |
+
258882
|
| 9 |
+
],
|
| 10 |
+
"temperature": 1.0,
|
| 11 |
+
"top_k": 64,
|
| 12 |
+
"top_p": 0.95,
|
| 13 |
+
"transformers_version": "5.14.1"
|
| 14 |
+
}
|
processor_config.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio_ms_per_token": 40,
|
| 3 |
+
"audio_seq_length": 750,
|
| 4 |
+
"feature_extractor": {
|
| 5 |
+
"audio_samples_per_token": 640,
|
| 6 |
+
"feature_extractor_type": "Gemma4UnifiedAudioFeatureExtractor",
|
| 7 |
+
"feature_size": 640,
|
| 8 |
+
"padding_side": "right",
|
| 9 |
+
"padding_value": 0.0,
|
| 10 |
+
"return_attention_mask": true,
|
| 11 |
+
"sampling_rate": 16000
|
| 12 |
+
},
|
| 13 |
+
"image_processor": {
|
| 14 |
+
"do_convert_rgb": true,
|
| 15 |
+
"do_normalize": false,
|
| 16 |
+
"do_rescale": true,
|
| 17 |
+
"do_resize": true,
|
| 18 |
+
"image_mean": [
|
| 19 |
+
0.0,
|
| 20 |
+
0.0,
|
| 21 |
+
0.0
|
| 22 |
+
],
|
| 23 |
+
"image_processor_type": "Gemma4UnifiedImageProcessor",
|
| 24 |
+
"image_std": [
|
| 25 |
+
1.0,
|
| 26 |
+
1.0,
|
| 27 |
+
1.0
|
| 28 |
+
],
|
| 29 |
+
"max_soft_tokens": 280,
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"pooling_kernel_size": 3,
|
| 32 |
+
"resample": 3,
|
| 33 |
+
"rescale_factor": 0.00392156862745098
|
| 34 |
+
},
|
| 35 |
+
"image_seq_length": 280,
|
| 36 |
+
"processor_class": "Gemma4UnifiedProcessor",
|
| 37 |
+
"video_processor": {
|
| 38 |
+
"do_convert_rgb": true,
|
| 39 |
+
"do_normalize": true,
|
| 40 |
+
"do_rescale": true,
|
| 41 |
+
"do_resize": true,
|
| 42 |
+
"do_sample_frames": true,
|
| 43 |
+
"image_mean": [
|
| 44 |
+
0.0,
|
| 45 |
+
0.0,
|
| 46 |
+
0.0
|
| 47 |
+
],
|
| 48 |
+
"image_std": [
|
| 49 |
+
1.0,
|
| 50 |
+
1.0,
|
| 51 |
+
1.0
|
| 52 |
+
],
|
| 53 |
+
"max_soft_tokens": 70,
|
| 54 |
+
"num_frames": 32,
|
| 55 |
+
"patch_size": 16,
|
| 56 |
+
"pooling_kernel_size": 3,
|
| 57 |
+
"resample": 3,
|
| 58 |
+
"rescale_factor": 0.00392156862745098,
|
| 59 |
+
"return_metadata": false,
|
| 60 |
+
"video_processor_type": "Gemma4UnifiedVideoProcessor"
|
| 61 |
+
}
|
| 62 |
+
}
|
prompt_format.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"style": "legacy",
|
| 3 |
+
"sep": "\n\n### Rewritten:\n\n",
|
| 4 |
+
"fingerprint": "cc51d66b4c593fbe",
|
| 5 |
+
"instr": "Rewrite the text below so it reads like a person wrote it, not a language model.\n\nReorganize it as you see fit. Vary sentence length on purpose. Cut hedging,\nthroat-clearing, and any sentence that only announces what comes next.\nPrefer the concrete word over the abstract one. It is fine to sound uneven.\n\nEvery fact, number, unit, date, name and quotation must survive unchanged.",
|
| 6 |
+
"chat": false,
|
| 7 |
+
"note": "推理时需 instr + draft + sep"
|
| 8 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:12bac982b793c44b03d52a250a9f0d0b666813da566b910c24a6da0695fd11e6
|
| 3 |
+
size 32170070
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio_token": "<|audio|>",
|
| 3 |
+
"backend": "tokenizers",
|
| 4 |
+
"boa_token": "<|audio>",
|
| 5 |
+
"boi_token": "<|image>",
|
| 6 |
+
"bos_token": "<bos>",
|
| 7 |
+
"eoa_token": "<audio|>",
|
| 8 |
+
"eoc_token": "<channel|>",
|
| 9 |
+
"eoi_token": "<image|>",
|
| 10 |
+
"eos_token": "<eos>",
|
| 11 |
+
"eot_token": "<turn|>",
|
| 12 |
+
"escape_token": "<|\"|>",
|
| 13 |
+
"etc_token": "<tool_call|>",
|
| 14 |
+
"etd_token": "<tool|>",
|
| 15 |
+
"etr_token": "<tool_response|>",
|
| 16 |
+
"extra_special_tokens": [
|
| 17 |
+
"<|video|>"
|
| 18 |
+
],
|
| 19 |
+
"image_token": "<|image|>",
|
| 20 |
+
"is_local": false,
|
| 21 |
+
"local_files_only": false,
|
| 22 |
+
"mask_token": "<mask>",
|
| 23 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 24 |
+
"model_specific_special_tokens": {
|
| 25 |
+
"audio_token": "<|audio|>",
|
| 26 |
+
"boa_token": "<|audio>",
|
| 27 |
+
"boi_token": "<|image>",
|
| 28 |
+
"eoa_token": "<audio|>",
|
| 29 |
+
"eoc_token": "<channel|>",
|
| 30 |
+
"eoi_token": "<image|>",
|
| 31 |
+
"eot_token": "<turn|>",
|
| 32 |
+
"escape_token": "<|\"|>",
|
| 33 |
+
"etc_token": "<tool_call|>",
|
| 34 |
+
"etd_token": "<tool|>",
|
| 35 |
+
"etr_token": "<tool_response|>",
|
| 36 |
+
"image_token": "<|image|>",
|
| 37 |
+
"soc_token": "<|channel>",
|
| 38 |
+
"sot_token": "<|turn>",
|
| 39 |
+
"stc_token": "<|tool_call>",
|
| 40 |
+
"std_token": "<|tool>",
|
| 41 |
+
"str_token": "<|tool_response>",
|
| 42 |
+
"think_token": "<|think|>"
|
| 43 |
+
},
|
| 44 |
+
"pad_token": "<pad>",
|
| 45 |
+
"padding_side": "left",
|
| 46 |
+
"processor_class": "Gemma4UnifiedProcessor",
|
| 47 |
+
"soc_token": "<|channel>",
|
| 48 |
+
"sot_token": "<|turn>",
|
| 49 |
+
"stc_token": "<|tool_call>",
|
| 50 |
+
"std_token": "<|tool>",
|
| 51 |
+
"str_token": "<|tool_response>",
|
| 52 |
+
"think_token": "<|think|>",
|
| 53 |
+
"tokenizer_class": "GemmaTokenizer",
|
| 54 |
+
"unk_token": "<unk>"
|
| 55 |
+
}
|