Hugging Face's logo

mlx-community
/
GLM-4.7-8bit-gs32

Text Generation
MLX
Safetensors
English
Chinese
glm4_moe
conversational
8-bit precision
Model card Files
xet
 Community
1
bibproj commited on
Commit
c05e758
·
verified ·
1 Parent(s): 0a04361

Upload

Browse files
Files changed (47) hide show
  1. .gitattributes +1 -0
  2. chat_template.jinja +86 -0
  3. config.json +53 -0
  4. generation_config.json +11 -0
  5. model-00001-of-00090.safetensors +3 -0
  6. model-00002-of-00090.safetensors +3 -0
  7. model-00003-of-00090.safetensors +3 -0
  8. model-00004-of-00090.safetensors +3 -0
  9. model-00005-of-00090.safetensors +3 -0
  10. model-00006-of-00090.safetensors +3 -0
  11. model-00007-of-00090.safetensors +3 -0
  12. model-00008-of-00090.safetensors +3 -0
  13. model-00009-of-00090.safetensors +3 -0
  14. model-00010-of-00090.safetensors +3 -0
  15. model-00011-of-00090.safetensors +3 -0
  16. model-00012-of-00090.safetensors +3 -0
  17. model-00013-of-00090.safetensors +3 -0
  18. model-00014-of-00090.safetensors +3 -0
  19. model-00015-of-00090.safetensors +3 -0
  20. model-00016-of-00090.safetensors +3 -0
  21. model-00017-of-00090.safetensors +3 -0
  22. model-00018-of-00090.safetensors +3 -0
  23. model-00019-of-00090.safetensors +3 -0
  24. model-00020-of-00090.safetensors +3 -0
  25. model-00021-of-00090.safetensors +3 -0
  26. model-00022-of-00090.safetensors +3 -0
  27. model-00023-of-00090.safetensors +3 -0
  28. model-00024-of-00090.safetensors +3 -0
  29. model-00025-of-00090.safetensors +3 -0
  30. model-00026-of-00090.safetensors +3 -0
  31. model-00027-of-00090.safetensors +3 -0
  32. model-00028-of-00090.safetensors +3 -0
  33. model-00029-of-00090.safetensors +3 -0
  34. model-00030-of-00090.safetensors +3 -0
  35. model-00031-of-00090.safetensors +3 -0
  36. model-00032-of-00090.safetensors +3 -0
  37. model-00033-of-00090.safetensors +3 -0
  38. model-00034-of-00090.safetensors +3 -0
  39. model-00035-of-00090.safetensors +3 -0
  40. model-00036-of-00090.safetensors +3 -0
  41. model-00037-of-00090.safetensors +3 -0
  42. model-00038-of-00090.safetensors +3 -0
  43. model-00039-of-00090.safetensors +3 -0
  44. model-00040-of-00090.safetensors +3 -0
  45. model.safetensors.index.json +0 -0
  46. tokenizer.json +3 -0
  47. tokenizer_config.json +61 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
chat_template.jinja ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [gMASK]<sop>
2
+ {%- if tools -%}
3
+ <|system|>
4
+ # Tools
5
+
6
+ You may call one or more functions to assist with the user query.
7
+
8
+ You are provided with function signatures within <tools></tools> XML tags:
9
+ <tools>
10
+ {% for tool in tools %}
11
+ {{ tool | tojson(ensure_ascii=False) }}
12
+ {% endfor %}
13
+ </tools>
14
+
15
+ For each function call, output the function name and arguments within the following XML format:
16
+ <tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
17
+ {%- macro visible_text(content) -%}
18
+ {%- if content is string -%}
19
+ {{- content }}
20
+ {%- elif content is iterable and content is not mapping -%}
21
+ {%- for item in content -%}
22
+ {%- if item is mapping and item.type == 'text' -%}
23
+ {{- item.text }}
24
+ {%- elif item is string -%}
25
+ {{- item }}
26
+ {%- endif -%}
27
+ {%- endfor -%}
28
+ {%- else -%}
29
+ {{- content }}
30
+ {%- endif -%}
31
+ {%- endmacro -%}
32
+ {%- set ns = namespace(last_user_index=-1) %}
33
+ {%- for m in messages %}
34
+ {%- if m.role == 'user' %}
35
+ {% set ns.last_user_index = loop.index0 -%}
36
+ {%- endif %}
37
+ {%- endfor %}
38
+ {% for m in messages %}
39
+ {%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}
40
+ {%- elif m.role == 'assistant' -%}
41
+ <|assistant|>
42
+ {%- set reasoning_content = '' %}
43
+ {%- set content = visible_text(m.content) %}
44
+ {%- if m.reasoning_content is string %}
45
+ {%- set reasoning_content = m.reasoning_content %}
46
+ {%- else %}
47
+ {%- if '</think>' in content %}
48
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
49
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
50
+ {%- endif %}
51
+ {%- endif %}
52
+ {%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content -%}
53
+ {{ '<think>' + reasoning_content.strip() + '</think>'}}
54
+ {%- else -%}
55
+ {{ '</think>' }}
56
+ {%- endif -%}
57
+ {%- if content.strip() -%}
58
+ {{ content.strip() }}
59
+ {%- endif -%}
60
+ {% if m.tool_calls %}
61
+ {% for tc in m.tool_calls %}
62
+ {%- if tc.function %}
63
+ {%- set tc = tc.function %}
64
+ {%- endif %}
65
+ {{- '<tool_call>' + tc.name -}}
66
+ {% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
67
+ {% endif %}
68
+ {%- elif m.role == 'tool' -%}
69
+ {%- if m.content is string -%}
70
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
71
+ {{- '<|observation|>' }}
72
+ {%- endif %}
73
+ {{- '<tool_response>' }}
74
+ {{- m.content }}
75
+ {{- '</tool_response>' }}
76
+ {%- else -%}
77
+ <|observation|>{% for tr in m.content %}
78
+ <tool_response>{{ tr.output if tr.output is defined else tr }}</tool_response>{% endfor -%}
79
+ {% endif -%}
80
+ {%- elif m.role == 'system' -%}
81
+ <|system|>{{ visible_text(m.content) }}
82
+ {%- endif -%}
83
+ {%- endfor -%}
84
+ {%- if add_generation_prompt -%}
85
+ <|assistant|>{{- '</think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
86
+ {%- endif -%}
config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm4MoeForCausalLM"
4
+ ],
5
+ "attention_bias": true,
6
+ "attention_dropout": 0.0,
7
+ "eos_token_id": [
8
+ 151329,
9
+ 151336,
10
+ 151338
11
+ ],
12
+ "first_k_dense_replace": 3,
13
+ "head_dim": 128,
14
+ "hidden_act": "silu",
15
+ "hidden_size": 5120,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 12288,
18
+ "max_position_embeddings": 202752,
19
+ "model_type": "glm4_moe",
20
+ "moe_intermediate_size": 1536,
21
+ "n_group": 1,
22
+ "n_routed_experts": 160,
23
+ "n_shared_experts": 1,
24
+ "norm_topk_prob": true,
25
+ "num_attention_heads": 96,
26
+ "num_experts_per_tok": 8,
27
+ "num_hidden_layers": 92,
28
+ "num_key_value_heads": 8,
29
+ "num_nextn_predict_layers": 1,
30
+ "pad_token_id": 151329,
31
+ "partial_rotary_factor": 0.5,
32
+ "quantization": {
33
+ "group_size": 32,
34
+ "bits": 8,
35
+ "mode": "affine"
36
+ },
37
+ "quantization_config": {
38
+ "group_size": 32,
39
+ "bits": 8,
40
+ "mode": "affine"
41
+ },
42
+ "rms_norm_eps": 1e-05,
43
+ "rope_scaling": null,
44
+ "rope_theta": 1000000,
45
+ "routed_scaling_factor": 2.5,
46
+ "tie_word_embeddings": false,
47
+ "topk_group": 1,
48
+ "torch_dtype": "bfloat16",
49
+ "transformers_version": "4.54.0",
50
+ "use_cache": true,
51
+ "use_qk_norm": true,
52
+ "vocab_size": 151552
53
+ }
generation_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": [
4
+ 151329,
5
+ 151336,
6
+ 151338
7
+ ],
8
+ "pad_token_id": 151329,
9
+ "temperature": 1.0,
10
+ "transformers_version": "4.56.2"
11
+ }
model-00001-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6c35adfaea827b9af7620af926baaa4c8d5c05ab674377a5aa702b6b6be3e53
3
+ size 4954712391
model-00002-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:328c4e54ab09eb1362cb881721d8cb07ac70cf540999139794e243c43198e5d2
3
+ size 4428322538
model-00003-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a68aa36e9f8055c20fa5b93b2dc8e72946faab1425da69aa49f14e2423fa765
3
+ size 4428322532
model-00004-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62f0b93d328cd29ac1da973fa84c35a91e81428e9bd970c1b12447f4ad7f785d
3
+ size 4428322548
model-00005-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c22ff87dcbc7e2541be910414c4bffbc72aec8c52c024b34225708f2f2eb9b3
3
+ size 4428322540
model-00006-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b5c8b0d4fdbe653ebcdaf281b8e7c06b02fe336f2cdec330411eb2c22e068b0
3
+ size 4428322542
model-00007-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0913f1cdeb88ca3b6eb1cd5a5cfc027d8d1770b908f5b44eefdb2b9cc4863ba4
3
+ size 4428322520
model-00008-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7963b6ecc09fbf4cd0723ef5ef74aa3e56dca801ac366e9193a400b09362429a
3
+ size 4428322571
model-00009-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75dccc5dc3dcc386720b6c2c806cdd4a9ef68a1327ee8605afc8ef691dbca900
3
+ size 4428322587
model-00010-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:705e9880e886f5c394e45c6938401572a0a050baae597fc68f20ff84812317c9
3
+ size 4428322539
model-00011-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be753f3c59bdb6281766c550742e8e24cd64ec440df0ee334e7d10cc0dbcfadb
3
+ size 4428322555
model-00012-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5678044f506c63f9df83583c8698f6ca3780eaa2a0ad9041caa9f0c66d14d847
3
+ size 4428322567
model-00013-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e4dd2052ca4af4289036bbd0cb0af3132547788723d16dc68e201165575a52a
3
+ size 4428322553
model-00014-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe4c39acccbc86e03b918a1a31dab0ee04a82af7bf1c98d3d66d4922ed65e979
3
+ size 4428322571
model-00015-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5c0deccf017d0c556fd2a9e3267ad62ee0936442d1c1ab0c4540fab6e53da07
3
+ size 4428322579
model-00016-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f21f8ffefcaf29fec405ff4884776abb4becffd3cabf95bc53ca954886d17c5
3
+ size 4428322551
model-00017-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66b89a17f7b6323ae6eba070d5520e89a8eeee9b2d4a5695d61a8279ba0c469b
3
+ size 4428322583
model-00018-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a433f69a895b07826b376fb56522c7e7898759a3d73be9c3cbe61331b16662c
3
+ size 4428322563
model-00019-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12b8b94c36fa55d4448047b6c8837a512c99597399a002751ddc03f8829dd235
3
+ size 4428322583
model-00020-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a153d65985545cc84e97ec27ec7e7afa49ac7e8636171cab4f8142ea392a57e7
3
+ size 4428322583
model-00021-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:245fdbc59b3012ccea1f996f4c2e233b475be3f6d3f247ada035940f8861dcac
3
+ size 4428322583
model-00022-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c2412b5b715c55e0010c928ba059074f80bcf836c20b23e933f491a4f14131c
3
+ size 4428322537
model-00023-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ced2ae542302b4efc1f7cdf5719631c01ba8f6bbe68de30964266b6a0fcf721
3
+ size 4428322583
model-00024-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eab56ebd4c5443b51d25780eea3428da737ba84a4de36d44ae0bda6b74c00d2e
3
+ size 4428322567
model-00025-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cec3fc1fb7f0027be2f0814f0351acbfa776ef845ccb240d5b5e1037fa890a64
3
+ size 4428322583
model-00026-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c9d057eecbff4fd32e388bf86b2d598a04d7c2cacc1f8d25f38766deacf288ce
3
+ size 4428322583
model-00027-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:35605080541bf3bddad1f2efeda489e551e6becf8938a46c7759da8c7b140491
3
+ size 4428322583
model-00028-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d2cb837c01541215038ff233193bcea33d9bbb71b19ad0f4de1f2990a57b7be
3
+ size 4428322553
model-00029-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75858aaa3394e5cfce5201b8eedb50834a67c9a49647f6db0ea6707d4a05322c
3
+ size 4428322579
model-00030-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:260b2d9bd60825c814c1daecdd4a833d4e470936cf1c2edf881b71537e447954
3
+ size 4428322583
model-00031-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e79624cc66dcda6755eaf18d7853e0818f40907d74a3045d9c508e6c6fa8590
3
+ size 4428322577
model-00032-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b5f12f3f03e1b5e54c6ee1af90250faa16150e8aac60a6b17930bc42d17522f
3
+ size 4428322577
model-00033-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66f6247497992395ee26ccac6a600cbe92c04b72f14aca3ea0fb138e8c00ada6
3
+ size 4428322583
model-00034-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ea6d99d16324ccf103095da155df498568b57ecf20a846ba8bd648cde164d90
3
+ size 4428322569
model-00035-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab80a7a9e6fdd53beab76eac4c870c3e55e5802b0abdd434477c52630e311ffa
3
+ size 4428322581
model-00036-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00029106cd3cd965cca203cc75b385b57d88fdde2afae28b4076f65afcca1c10
3
+ size 4428322581
model-00037-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08a3a8d369a43a4e0e6a4924fc9ba1f0aaff6d49b8a6f8a57ebd63c9b58c5350
3
+ size 4428322531
model-00038-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb7003462e9d38176e2b5b71900872c5578271c3ff4eba5e9ae0313d0416d9c2
3
+ size 4428322583
model-00039-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:524440e5cb6721d7b986425504ba8f334d88395497f99fa60bcc9ad8710143f6
3
+ size 4428322583
model-00040-of-00090.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9761371d6b17b887a9a4435c22666b03c6c89cb2445e9020fc602e52daaa428d
3
+ size 4428322583
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0ccf2252fe9cd23ada23a829ec409aab397dbd2ac4f372d3a1a23d1f7c72d6b
3
+ size 19970686
tokenizer_config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "[MASK]",
5
+ "[gMASK]",
6
+ "[sMASK]",
7
+ "<sop>",
8
+ "<eop>",
9
+ "<|system|>",
10
+ "<|user|>",
11
+ "<|assistant|>",
12
+ "<|observation|>",
13
+ "<|begin_of_image|>",
14
+ "<|end_of_image|>",
15
+ "<|begin_of_video|>",
16
+ "<|end_of_video|>",
17
+ "<|begin_of_audio|>",
18
+ "<|end_of_audio|>",
19
+ "<|begin_of_transcription|>",
20
+ "<|end_of_transcription|>",
21
+ "<|code_prefix|>",
22
+ "<|code_middle|>",
23
+ "<|code_suffix|>",
24
+ "/nothink"
25
+ ],
26
+ "backend": "tokenizers",
27
+ "clean_up_tokenization_spaces": false,
28
+ "do_lower_case": false,
29
+ "eos_token": "<|endoftext|>",
30
+ "extra_special_tokens": [
31
+ "<|endoftext|>",
32
+ "[MASK]",
33
+ "[gMASK]",
34
+ "[sMASK]",
35
+ "<sop>",
36
+ "<eop>",
37
+ "<|system|>",
38
+ "<|user|>",
39
+ "<|assistant|>",
40
+ "<|observation|>",
41
+ "<|begin_of_image|>",
42
+ "<|end_of_image|>",
43
+ "<|begin_of_video|>",
44
+ "<|end_of_video|>",
45
+ "<|begin_of_audio|>",
46
+ "<|end_of_audio|>",
47
+ "<|begin_of_transcription|>",
48
+ "<|end_of_transcription|>",
49
+ "<|code_prefix|>",
50
+ "<|code_middle|>",
51
+ "<|code_suffix|>",
52
+ "/nothink"
53
+ ],
54
+ "is_local": true,
55
+ "model_max_length": 128000,
56
+ "model_specific_special_tokens": {},
57
+ "pad_token": "<|endoftext|>",
58
+ "padding_side": "left",
59
+ "remove_space": false,
60
+ "tokenizer_class": "TokenizersBackend"
61
+ }