VictorM-Coder commited on
Commit
2434cc6
·
verified ·
1 Parent(s): 82c786b

Add all tokenizer, processor, and prompt configs

Browse files
.gitattributes CHANGED
@@ -33,3 +33,38 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ gguf/humanizer-gemma-4-e4b-bf16.gguf filter=lfs diff=lfs merge=lfs -text
38
+ gguf/humanizer-gemma-4-e4b-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
39
+ gguf/humanizer-gemma-4-e4b-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text
40
+ gguf/humanizer-gemma-4-e4b-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
41
+ gguf/humanizer-gemma-4-e4b-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
42
+ lite/humanizer-lite-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
43
+ lite/humanizer-lite-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
44
+ lite/humanizer-lite-bf16.gguf filter=lfs diff=lfs merge=lfs -text
45
+ humanizer-12b-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
46
+ humanizer-12b-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
47
+ assets/results-detector-en.png filter=lfs diff=lfs merge=lfs -text
48
+ assets/training-zh.png filter=lfs diff=lfs merge=lfs -text
49
+ assets/app-showcase-zh.png filter=lfs diff=lfs merge=lfs -text
50
+ assets/banner-zh.png filter=lfs diff=lfs merge=lfs -text
51
+ assets/compare-en-email.png filter=lfs diff=lfs merge=lfs -text
52
+ assets/compare-zh-zhihu.png filter=lfs diff=lfs merge=lfs -text
53
+ assets/training-en.png filter=lfs diff=lfs merge=lfs -text
54
+ assets/results-detector-zh.png filter=lfs diff=lfs merge=lfs -text
55
+ assets/banner-en.png filter=lfs diff=lfs merge=lfs -text
56
+ assets/app-showcase-en.png filter=lfs diff=lfs merge=lfs -text
57
+ assets/compare-en-reddit.png filter=lfs diff=lfs merge=lfs -text
58
+ assets/results-fidelity-en.png filter=lfs diff=lfs merge=lfs -text
59
+ assets/compare-zh-email.png filter=lfs diff=lfs merge=lfs -text
60
+ assets/app-en.png filter=lfs diff=lfs merge=lfs -text
61
+ assets/app-zh.png filter=lfs diff=lfs merge=lfs -text
62
+ assets/results-fidelity-zh.png filter=lfs diff=lfs merge=lfs -text
63
+ humanizer-12b-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
64
+ assets/compare-en-forum.png filter=lfs diff=lfs merge=lfs -text
65
+ humanizer-12b-bf16.gguf filter=lfs diff=lfs merge=lfs -text
66
+ humanizer-12b-IQ2_XS-QAT.gguf filter=lfs diff=lfs merge=lfs -text
67
+ humanizer-12b-Q3-QAT.gguf filter=lfs diff=lfs merge=lfs -text
68
+ assets/quant-top1-en.png filter=lfs diff=lfs merge=lfs -text
69
+ assets/eval-en.png filter=lfs diff=lfs merge=lfs -text
70
+ assets/eval-zh.png filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Gemma4UnifiedForConditionalGeneration"
4
+ ],
5
+ "audio_config": {
6
+ "_name_or_path": "",
7
+ "architectures": null,
8
+ "audio_embed_dim": 640,
9
+ "chunk_size_feed_forward": 0,
10
+ "dtype": "bfloat16",
11
+ "id2label": {
12
+ "0": "LABEL_0",
13
+ "1": "LABEL_1"
14
+ },
15
+ "initializer_range": 0.02,
16
+ "is_encoder_decoder": false,
17
+ "label2id": {
18
+ "LABEL_0": 0,
19
+ "LABEL_1": 1
20
+ },
21
+ "model_type": "gemma4_unified_audio",
22
+ "output_attentions": false,
23
+ "output_hidden_states": false,
24
+ "problem_type": null,
25
+ "return_dict": true,
26
+ "rms_norm_eps": 1e-06
27
+ },
28
+ "audio_token_id": 258881,
29
+ "boa_token_id": 256000,
30
+ "boi_token_id": 255999,
31
+ "dtype": "bfloat16",
32
+ "eoa_token_index": 258883,
33
+ "eoi_token_id": 258882,
34
+ "image_token_id": 258880,
35
+ "initializer_range": 0.02,
36
+ "model_type": "gemma4_unified",
37
+ "text_config": {
38
+ "attention_bias": false,
39
+ "attention_dropout": 0.0,
40
+ "attention_k_eq_v": true,
41
+ "bos_token_id": 2,
42
+ "dtype": "bfloat16",
43
+ "enable_moe_block": false,
44
+ "eos_token_id": 1,
45
+ "final_logit_softcapping": 30.0,
46
+ "global_head_dim": 512,
47
+ "head_dim": 256,
48
+ "hidden_activation": "gelu_pytorch_tanh",
49
+ "hidden_size": 3840,
50
+ "hidden_size_per_layer_input": 0,
51
+ "initializer_range": 0.02,
52
+ "intermediate_size": 15360,
53
+ "layer_types": [
54
+ "sliding_attention",
55
+ "sliding_attention",
56
+ "sliding_attention",
57
+ "sliding_attention",
58
+ "sliding_attention",
59
+ "full_attention",
60
+ "sliding_attention",
61
+ "sliding_attention",
62
+ "sliding_attention",
63
+ "sliding_attention",
64
+ "sliding_attention",
65
+ "full_attention",
66
+ "sliding_attention",
67
+ "sliding_attention",
68
+ "sliding_attention",
69
+ "sliding_attention",
70
+ "sliding_attention",
71
+ "full_attention",
72
+ "sliding_attention",
73
+ "sliding_attention",
74
+ "sliding_attention",
75
+ "sliding_attention",
76
+ "sliding_attention",
77
+ "full_attention",
78
+ "sliding_attention",
79
+ "sliding_attention",
80
+ "sliding_attention",
81
+ "sliding_attention",
82
+ "sliding_attention",
83
+ "full_attention",
84
+ "sliding_attention",
85
+ "sliding_attention",
86
+ "sliding_attention",
87
+ "sliding_attention",
88
+ "sliding_attention",
89
+ "full_attention",
90
+ "sliding_attention",
91
+ "sliding_attention",
92
+ "sliding_attention",
93
+ "sliding_attention",
94
+ "sliding_attention",
95
+ "full_attention",
96
+ "sliding_attention",
97
+ "sliding_attention",
98
+ "sliding_attention",
99
+ "sliding_attention",
100
+ "sliding_attention",
101
+ "full_attention"
102
+ ],
103
+ "max_position_embeddings": 262144,
104
+ "model_type": "gemma4_unified_text",
105
+ "moe_intermediate_size": null,
106
+ "num_attention_heads": 16,
107
+ "num_experts": null,
108
+ "num_global_key_value_heads": 1,
109
+ "num_hidden_layers": 48,
110
+ "num_key_value_heads": 8,
111
+ "num_kv_shared_layers": 0,
112
+ "pad_token_id": 0,
113
+ "rms_norm_eps": 1e-06,
114
+ "rope_parameters": {
115
+ "full_attention": {
116
+ "partial_rotary_factor": 0.25,
117
+ "rope_theta": 1000000.0,
118
+ "rope_type": "proportional"
119
+ },
120
+ "sliding_attention": {
121
+ "rope_theta": 10000.0,
122
+ "rope_type": "default"
123
+ }
124
+ },
125
+ "sliding_window": 1024,
126
+ "tie_word_embeddings": true,
127
+ "top_k_experts": null,
128
+ "use_bidirectional_attention": "vision",
129
+ "use_cache": true,
130
+ "use_double_wide_mlp": false,
131
+ "vocab_size": 262144,
132
+ "vocab_size_per_layer_input": 262144
133
+ },
134
+ "tie_word_embeddings": true,
135
+ "transformers_version": "5.14.1",
136
+ "video_token_id": 258884,
137
+ "vision_config": {
138
+ "_name_or_path": "",
139
+ "architectures": null,
140
+ "chunk_size_feed_forward": 0,
141
+ "dtype": "bfloat16",
142
+ "id2label": {
143
+ "0": "LABEL_0",
144
+ "1": "LABEL_1"
145
+ },
146
+ "initializer_range": 0.02,
147
+ "is_encoder_decoder": false,
148
+ "label2id": {
149
+ "LABEL_0": 0,
150
+ "LABEL_1": 1
151
+ },
152
+ "mm_embed_dim": 3840,
153
+ "mm_posemb_size": 1120,
154
+ "model_type": "gemma4_unified_vision",
155
+ "num_soft_tokens": 280,
156
+ "output_attentions": false,
157
+ "output_hidden_states": false,
158
+ "output_proj_dims": 3840,
159
+ "patch_size": 16,
160
+ "pooling_kernel_size": 3,
161
+ "problem_type": null,
162
+ "return_dict": true,
163
+ "rms_norm_eps": 1e-06
164
+ }
165
+ }
generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 2,
3
+ "do_sample": true,
4
+ "eos_token_id": 1,
5
+ "pad_token_id": 0,
6
+ "suppress_tokens": [
7
+ 258883,
8
+ 258882
9
+ ],
10
+ "temperature": 1.0,
11
+ "top_k": 64,
12
+ "top_p": 0.95,
13
+ "transformers_version": "5.14.1"
14
+ }
processor_config.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio_ms_per_token": 40,
3
+ "audio_seq_length": 750,
4
+ "feature_extractor": {
5
+ "audio_samples_per_token": 640,
6
+ "feature_extractor_type": "Gemma4UnifiedAudioFeatureExtractor",
7
+ "feature_size": 640,
8
+ "padding_side": "right",
9
+ "padding_value": 0.0,
10
+ "return_attention_mask": true,
11
+ "sampling_rate": 16000
12
+ },
13
+ "image_processor": {
14
+ "do_convert_rgb": true,
15
+ "do_normalize": false,
16
+ "do_rescale": true,
17
+ "do_resize": true,
18
+ "image_mean": [
19
+ 0.0,
20
+ 0.0,
21
+ 0.0
22
+ ],
23
+ "image_processor_type": "Gemma4UnifiedImageProcessor",
24
+ "image_std": [
25
+ 1.0,
26
+ 1.0,
27
+ 1.0
28
+ ],
29
+ "max_soft_tokens": 280,
30
+ "patch_size": 16,
31
+ "pooling_kernel_size": 3,
32
+ "resample": 3,
33
+ "rescale_factor": 0.00392156862745098
34
+ },
35
+ "image_seq_length": 280,
36
+ "processor_class": "Gemma4UnifiedProcessor",
37
+ "video_processor": {
38
+ "do_convert_rgb": true,
39
+ "do_normalize": true,
40
+ "do_rescale": true,
41
+ "do_resize": true,
42
+ "do_sample_frames": true,
43
+ "image_mean": [
44
+ 0.0,
45
+ 0.0,
46
+ 0.0
47
+ ],
48
+ "image_std": [
49
+ 1.0,
50
+ 1.0,
51
+ 1.0
52
+ ],
53
+ "max_soft_tokens": 70,
54
+ "num_frames": 32,
55
+ "patch_size": 16,
56
+ "pooling_kernel_size": 3,
57
+ "resample": 3,
58
+ "rescale_factor": 0.00392156862745098,
59
+ "return_metadata": false,
60
+ "video_processor_type": "Gemma4UnifiedVideoProcessor"
61
+ }
62
+ }
prompt_format.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "style": "legacy",
3
+ "sep": "\n\n### Rewritten:\n\n",
4
+ "fingerprint": "cc51d66b4c593fbe",
5
+ "instr": "Rewrite the text below so it reads like a person wrote it, not a language model.\n\nReorganize it as you see fit. Vary sentence length on purpose. Cut hedging,\nthroat-clearing, and any sentence that only announces what comes next.\nPrefer the concrete word over the abstract one. It is fine to sound uneven.\n\nEvery fact, number, unit, date, name and quotation must survive unchanged.",
6
+ "chat": false,
7
+ "note": "推理时需 instr + draft + sep"
8
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12bac982b793c44b03d52a250a9f0d0b666813da566b910c24a6da0695fd11e6
3
+ size 32170070
tokenizer_config.json ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio_token": "<|audio|>",
3
+ "backend": "tokenizers",
4
+ "boa_token": "<|audio>",
5
+ "boi_token": "<|image>",
6
+ "bos_token": "<bos>",
7
+ "eoa_token": "<audio|>",
8
+ "eoc_token": "<channel|>",
9
+ "eoi_token": "<image|>",
10
+ "eos_token": "<eos>",
11
+ "eot_token": "<turn|>",
12
+ "escape_token": "<|\"|>",
13
+ "etc_token": "<tool_call|>",
14
+ "etd_token": "<tool|>",
15
+ "etr_token": "<tool_response|>",
16
+ "extra_special_tokens": [
17
+ "<|video|>"
18
+ ],
19
+ "image_token": "<|image|>",
20
+ "is_local": false,
21
+ "local_files_only": false,
22
+ "mask_token": "<mask>",
23
+ "model_max_length": 1000000000000000019884624838656,
24
+ "model_specific_special_tokens": {
25
+ "audio_token": "<|audio|>",
26
+ "boa_token": "<|audio>",
27
+ "boi_token": "<|image>",
28
+ "eoa_token": "<audio|>",
29
+ "eoc_token": "<channel|>",
30
+ "eoi_token": "<image|>",
31
+ "eot_token": "<turn|>",
32
+ "escape_token": "<|\"|>",
33
+ "etc_token": "<tool_call|>",
34
+ "etd_token": "<tool|>",
35
+ "etr_token": "<tool_response|>",
36
+ "image_token": "<|image|>",
37
+ "soc_token": "<|channel>",
38
+ "sot_token": "<|turn>",
39
+ "stc_token": "<|tool_call>",
40
+ "std_token": "<|tool>",
41
+ "str_token": "<|tool_response>",
42
+ "think_token": "<|think|>"
43
+ },
44
+ "pad_token": "<pad>",
45
+ "padding_side": "left",
46
+ "processor_class": "Gemma4UnifiedProcessor",
47
+ "soc_token": "<|channel>",
48
+ "sot_token": "<|turn>",
49
+ "stc_token": "<|tool_call>",
50
+ "std_token": "<|tool>",
51
+ "str_token": "<|tool_response>",
52
+ "think_token": "<|think|>",
53
+ "tokenizer_class": "GemmaTokenizer",
54
+ "unk_token": "<unk>"
55
+ }