Hugging Face's logo

MuXodious
/
GLM-4.7-Flash-absolute-heresy

Text Generation
Transformers
Safetensors
English
Chinese
glm4_moe_lite
heretic
uncensored
decensored
abliterated
conversational
Model card Files
xet
 Community
4
MuXodious commited on
Commit
7ddffe2
·
verified ·
1 Parent(s): 867df97

Aligner

Browse files
Files changed (2) hide show
  1. config.json +16 -74
  2. tokenizer_config.json +312 -3
config.json CHANGED
@@ -4,101 +4,43 @@
4
  ],
5
  "attention_bias": false,
6
  "attention_dropout": 0.0,
7
- "bos_token_id": 0,
8
- "dtype": "bfloat16",
9
  "eos_token_id": [
10
  154820,
11
  154827,
12
  154829
13
  ],
14
- "first_k_dense_replace": 1,
15
- "head_dim": 64,
16
  "hidden_act": "silu",
17
  "hidden_size": 2048,
18
- "initializer_range": 0.02,
19
  "intermediate_size": 10240,
20
- "kv_lora_rank": 512,
21
  "max_position_embeddings": 202752,
22
- "mlp_layer_types": [
23
- "dense",
24
- "sparse",
25
- "sparse",
26
- "sparse",
27
- "sparse",
28
- "sparse",
29
- "sparse",
30
- "sparse",
31
- "sparse",
32
- "sparse",
33
- "sparse",
34
- "sparse",
35
- "sparse",
36
- "sparse",
37
- "sparse",
38
- "sparse",
39
- "sparse",
40
- "sparse",
41
- "sparse",
42
- "sparse",
43
- "sparse",
44
- "sparse",
45
- "sparse",
46
- "sparse",
47
- "sparse",
48
- "sparse",
49
- "sparse",
50
- "sparse",
51
- "sparse",
52
- "sparse",
53
- "sparse",
54
- "sparse",
55
- "sparse",
56
- "sparse",
57
- "sparse",
58
- "sparse",
59
- "sparse",
60
- "sparse",
61
- "sparse",
62
- "sparse",
63
- "sparse",
64
- "sparse",
65
- "sparse",
66
- "sparse",
67
- "sparse",
68
- "sparse",
69
- "sparse"
70
- ],
71
  "model_type": "glm4_moe_lite",
72
  "moe_intermediate_size": 1536,
 
 
 
73
  "n_group": 1,
 
74
  "n_routed_experts": 64,
75
  "n_shared_experts": 1,
76
- "norm_topk_prob": true,
77
- "num_attention_heads": 20,
78
  "num_experts_per_tok": 4,
 
79
  "num_hidden_layers": 47,
80
  "num_key_value_heads": 20,
81
  "num_nextn_predict_layers": 1,
82
- "pad_token_id": 154820,
83
  "partial_rotary_factor": 1.0,
84
- "pretraining_tp": 1,
85
- "q_lora_rank": 768,
86
- "qk_head_dim": 256,
87
- "qk_nope_head_dim": 192,
88
- "qk_rope_head_dim": 64,
89
  "rms_norm_eps": 1e-05,
90
- "rope_interleave": true,
91
- "rope_parameters": {
92
- "partial_rotary_factor": 1.0,
93
- "rope_theta": 1000000,
94
- "rope_type": "default"
95
- },
96
- "routed_scaling_factor": 1.8,
97
  "tie_word_embeddings": false,
98
- "topk_group": 1,
99
- "topk_method": "noaux_tc",
100
  "transformers_version": "5.2.0.dev0",
101
- "use_cache": true,
 
 
 
102
  "v_head_dim": 256,
103
- "vocab_size": 154880
 
104
  }
 
4
  ],
5
  "attention_bias": false,
6
  "attention_dropout": 0.0,
7
+ "pad_token_id": 154820,
 
8
  "eos_token_id": [
9
  154820,
10
  154827,
11
  154829
12
  ],
 
 
13
  "hidden_act": "silu",
14
  "hidden_size": 2048,
 
15
  "intermediate_size": 10240,
 
16
  "max_position_embeddings": 202752,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  "model_type": "glm4_moe_lite",
18
  "moe_intermediate_size": 1536,
19
+ "topk_method": "noaux_tc",
20
+ "norm_topk_prob": true,
21
+ "num_attention_heads": 20,
22
  "n_group": 1,
23
+ "topk_group": 1,
24
  "n_routed_experts": 64,
25
  "n_shared_experts": 1,
26
+ "routed_scaling_factor": 1.8,
 
27
  "num_experts_per_tok": 4,
28
+ "first_k_dense_replace": 1,
29
  "num_hidden_layers": 47,
30
  "num_key_value_heads": 20,
31
  "num_nextn_predict_layers": 1,
 
32
  "partial_rotary_factor": 1.0,
 
 
 
 
 
33
  "rms_norm_eps": 1e-05,
34
+ "rope_scaling": null,
35
+ "rope_theta": 1000000,
 
 
 
 
 
36
  "tie_word_embeddings": false,
37
+ "dtype": "bfloat16",
 
38
  "transformers_version": "5.2.0.dev0",
39
+ "q_lora_rank": 768,
40
+ "kv_lora_rank": 512,
41
+ "qk_nope_head_dim": 192,
42
+ "qk_rope_head_dim": 64,
43
  "v_head_dim": 256,
44
+ "vocab_size": 154880,
45
+ "scoring_func": "sigmoid"
46
  }
tokenizer_config.json CHANGED
@@ -1,12 +1,321 @@
1
  {
2
- "backend": "tokenizers",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  "clean_up_tokenization_spaces": false,
4
  "do_lower_case": false,
5
  "eos_token": "<|endoftext|>",
6
- "is_local": false,
7
  "model_max_length": 128000,
8
  "pad_token": "<|endoftext|>",
9
  "padding_side": "left",
10
  "remove_space": false,
11
- "tokenizer_class": "TokenizersBackend"
12
  }
 
1
  {
2
+ "added_tokens_decoder": {
3
+ "154820": {
4
+ "content": "<|endoftext|>",
5
+ "single_word": false,
6
+ "lstrip": false,
7
+ "rstrip": false,
8
+ "normalized": false,
9
+ "special": true
10
+ },
11
+ "154821": {
12
+ "content": "[MASK]",
13
+ "single_word": false,
14
+ "lstrip": false,
15
+ "rstrip": false,
16
+ "normalized": false,
17
+ "special": true
18
+ },
19
+ "154822": {
20
+ "content": "[gMASK]",
21
+ "single_word": false,
22
+ "lstrip": false,
23
+ "rstrip": false,
24
+ "normalized": false,
25
+ "special": true
26
+ },
27
+ "154823": {
28
+ "content": "[sMASK]",
29
+ "single_word": false,
30
+ "lstrip": false,
31
+ "rstrip": false,
32
+ "normalized": false,
33
+ "special": true
34
+ },
35
+ "154824": {
36
+ "content": "<sop>",
37
+ "single_word": false,
38
+ "lstrip": false,
39
+ "rstrip": false,
40
+ "normalized": false,
41
+ "special": true
42
+ },
43
+ "154825": {
44
+ "content": "<eop>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ "154826": {
52
+ "content": "<|system|>",
53
+ "single_word": false,
54
+ "lstrip": false,
55
+ "rstrip": false,
56
+ "normalized": false,
57
+ "special": true
58
+ },
59
+ "154827": {
60
+ "content": "<|user|>",
61
+ "single_word": false,
62
+ "lstrip": false,
63
+ "rstrip": false,
64
+ "normalized": false,
65
+ "special": true
66
+ },
67
+ "154828": {
68
+ "content": "<|assistant|>",
69
+ "single_word": false,
70
+ "lstrip": false,
71
+ "rstrip": false,
72
+ "normalized": false,
73
+ "special": true
74
+ },
75
+ "154829": {
76
+ "content": "<|observation|>",
77
+ "single_word": false,
78
+ "lstrip": false,
79
+ "rstrip": false,
80
+ "normalized": false,
81
+ "special": true
82
+ },
83
+ "154830": {
84
+ "content": "<|begin_of_image|>",
85
+ "single_word": false,
86
+ "lstrip": false,
87
+ "rstrip": false,
88
+ "normalized": false,
89
+ "special": true
90
+ },
91
+ "154831": {
92
+ "content": "<|end_of_image|>",
93
+ "single_word": false,
94
+ "lstrip": false,
95
+ "rstrip": false,
96
+ "normalized": false,
97
+ "special": true
98
+ },
99
+ "154832": {
100
+ "content": "<|begin_of_video|>",
101
+ "single_word": false,
102
+ "lstrip": false,
103
+ "rstrip": false,
104
+ "normalized": false,
105
+ "special": true
106
+ },
107
+ "154833": {
108
+ "content": "<|end_of_video|>",
109
+ "single_word": false,
110
+ "lstrip": false,
111
+ "rstrip": false,
112
+ "normalized": false,
113
+ "special": true
114
+ },
115
+ "154834": {
116
+ "content": "<|begin_of_audio|>",
117
+ "single_word": false,
118
+ "lstrip": false,
119
+ "rstrip": false,
120
+ "normalized": false,
121
+ "special": true
122
+ },
123
+ "154835": {
124
+ "content": "<|end_of_audio|>",
125
+ "single_word": false,
126
+ "lstrip": false,
127
+ "rstrip": false,
128
+ "normalized": false,
129
+ "special": true
130
+ },
131
+ "154836": {
132
+ "content": "<|begin_of_transcription|>",
133
+ "single_word": false,
134
+ "lstrip": false,
135
+ "rstrip": false,
136
+ "normalized": false,
137
+ "special": true
138
+ },
139
+ "154837": {
140
+ "content": "<|end_of_transcription|>",
141
+ "single_word": false,
142
+ "lstrip": false,
143
+ "rstrip": false,
144
+ "normalized": false,
145
+ "special": true
146
+ },
147
+ "154838": {
148
+ "content": "<|code_prefix|>",
149
+ "single_word": false,
150
+ "lstrip": false,
151
+ "rstrip": false,
152
+ "normalized": false,
153
+ "special": false
154
+ },
155
+ "154839": {
156
+ "content": "<|code_middle|>",
157
+ "single_word": false,
158
+ "lstrip": false,
159
+ "rstrip": false,
160
+ "normalized": false,
161
+ "special": false
162
+ },
163
+ "154840": {
164
+ "content": "<|code_suffix|>",
165
+ "single_word": false,
166
+ "lstrip": false,
167
+ "rstrip": false,
168
+ "normalized": false,
169
+ "special": false
170
+ },
171
+ "154841": {
172
+ "content": "<think>",
173
+ "single_word": false,
174
+ "lstrip": false,
175
+ "rstrip": false,
176
+ "normalized": false,
177
+ "special": false
178
+ },
179
+ "154842": {
180
+ "content": "</think>",
181
+ "single_word": false,
182
+ "lstrip": false,
183
+ "rstrip": false,
184
+ "normalized": false,
185
+ "special": false
186
+ },
187
+ "154843": {
188
+ "content": "<tool_call>",
189
+ "single_word": false,
190
+ "lstrip": false,
191
+ "rstrip": false,
192
+ "normalized": false,
193
+ "special": false
194
+ },
195
+ "154844": {
196
+ "content": "</tool_call>",
197
+ "single_word": false,
198
+ "lstrip": false,
199
+ "rstrip": false,
200
+ "normalized": false,
201
+ "special": false
202
+ },
203
+ "154845": {
204
+ "content": "<tool_response>",
205
+ "single_word": false,
206
+ "lstrip": false,
207
+ "rstrip": false,
208
+ "normalized": false,
209
+ "special": false
210
+ },
211
+ "154846": {
212
+ "content": "</tool_response>",
213
+ "single_word": false,
214
+ "lstrip": false,
215
+ "rstrip": false,
216
+ "normalized": false,
217
+ "special": false
218
+ },
219
+ "154847": {
220
+ "content": "<arg_key>",
221
+ "single_word": false,
222
+ "lstrip": false,
223
+ "rstrip": false,
224
+ "normalized": false,
225
+ "special": false
226
+ },
227
+ "154848": {
228
+ "content": "</arg_key>",
229
+ "single_word": false,
230
+ "lstrip": false,
231
+ "rstrip": false,
232
+ "normalized": false,
233
+ "special": false
234
+ },
235
+ "154849": {
236
+ "content": "<arg_value>",
237
+ "single_word": false,
238
+ "lstrip": false,
239
+ "rstrip": false,
240
+ "normalized": false,
241
+ "special": false
242
+ },
243
+ "154850": {
244
+ "content": "</arg_value>",
245
+ "single_word": false,
246
+ "lstrip": false,
247
+ "rstrip": false,
248
+ "normalized": false,
249
+ "special": false
250
+ },
251
+ "154851": {
252
+ "content": "/nothink",
253
+ "single_word": false,
254
+ "lstrip": false,
255
+ "rstrip": false,
256
+ "normalized": false,
257
+ "special": false
258
+ },
259
+ "154852": {
260
+ "content": "<|begin_of_box|>",
261
+ "single_word": false,
262
+ "lstrip": false,
263
+ "rstrip": false,
264
+ "normalized": false,
265
+ "special": false
266
+ },
267
+ "154853": {
268
+ "content": "<|end_of_box|>",
269
+ "single_word": false,
270
+ "lstrip": false,
271
+ "rstrip": false,
272
+ "normalized": false,
273
+ "special": false
274
+ },
275
+ "154854": {
276
+ "content": "<|image|>",
277
+ "single_word": false,
278
+ "lstrip": false,
279
+ "rstrip": false,
280
+ "normalized": false,
281
+ "special": false
282
+ },
283
+ "154855": {
284
+ "content": "<|video|>",
285
+ "single_word": false,
286
+ "lstrip": false,
287
+ "rstrip": false,
288
+ "normalized": false,
289
+ "special": false
290
+ }
291
+ },
292
+ "additional_special_tokens": [
293
+ "<|endoftext|>",
294
+ "[MASK]",
295
+ "[gMASK]",
296
+ "[sMASK]",
297
+ "<sop>",
298
+ "<eop>",
299
+ "<|system|>",
300
+ "<|user|>",
301
+ "<|assistant|>",
302
+ "<|observation|>",
303
+ "<|begin_of_image|>",
304
+ "<|end_of_image|>",
305
+ "<|begin_of_video|>",
306
+ "<|end_of_video|>",
307
+ "<|begin_of_audio|>",
308
+ "<|end_of_audio|>",
309
+ "<|begin_of_transcription|>",
310
+ "<|end_of_transcription|>"
311
+ ],
312
  "clean_up_tokenization_spaces": false,
313
  "do_lower_case": false,
314
  "eos_token": "<|endoftext|>",
315
+ "extra_special_tokens": {},
316
  "model_max_length": 128000,
317
  "pad_token": "<|endoftext|>",
318
  "padding_side": "left",
319
  "remove_space": false,
320
+ "tokenizer_class": "PreTrainedTokenizer"
321
  }