Hugging Face's logo

spicyneuron
/
Qwen3.5-9B-MLX-5.6bit-vision

Image-Text-to-Text
MLX
Safetensors
qwen3_5
conversational
4-bit precision
Model card Files
xet
 Community
spicyneuron commited on
Commit
4a44fa7
·
verified ·
1 Parent(s): 3e47c16

Add files using upload-large-folder tool

Browse files
Files changed (13) hide show
  1. .gitattributes +1 -0
  2. README.md +9 -0
  3. chat_template.jinja +154 -0
  4. config.json +1243 -0
  5. model-00001-of-00002.safetensors +3 -0
  6. model-00002-of-00002.safetensors +3 -0
  7. model.safetensors.index.json +0 -0
  8. preprocessor_config.json +21 -0
  9. processor_config.json +50 -0
  10. tokenizer.json +3 -0
  11. tokenizer_config.json +33 -0
  12. video_preprocessor_config.json +21 -0
  13. vocab.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: mlx
3
+ license: apache-2.0
4
+ license_link: https://huggingface.co/Qwen/Qwen3.5-9B/blob/main/LICENSE
5
+ pipeline_tag: image-text-to-text
6
+ base_model: Qwen/Qwen3.5-9B
7
+ tags:
8
+ - mlx
9
+ ---
chat_template.jinja ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- set image_count = namespace(value=0) %}
2
+ {%- set video_count = namespace(value=0) %}
3
+ {%- macro render_content(content, do_vision_count, is_system_content=false) %}
4
+ {%- if content is string %}
5
+ {{- content }}
6
+ {%- elif content is iterable and content is not mapping %}
7
+ {%- for item in content %}
8
+ {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
9
+ {%- if is_system_content %}
10
+ {{- raise_exception('System message cannot contain images.') }}
11
+ {%- endif %}
12
+ {%- if do_vision_count %}
13
+ {%- set image_count.value = image_count.value + 1 %}
14
+ {%- endif %}
15
+ {%- if add_vision_id %}
16
+ {{- 'Picture ' ~ image_count.value ~ ': ' }}
17
+ {%- endif %}
18
+ {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
19
+ {%- elif 'video' in item or item.type == 'video' %}
20
+ {%- if is_system_content %}
21
+ {{- raise_exception('System message cannot contain videos.') }}
22
+ {%- endif %}
23
+ {%- if do_vision_count %}
24
+ {%- set video_count.value = video_count.value + 1 %}
25
+ {%- endif %}
26
+ {%- if add_vision_id %}
27
+ {{- 'Video ' ~ video_count.value ~ ': ' }}
28
+ {%- endif %}
29
+ {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
30
+ {%- elif 'text' in item %}
31
+ {{- item.text }}
32
+ {%- else %}
33
+ {{- raise_exception('Unexpected item type in content.') }}
34
+ {%- endif %}
35
+ {%- endfor %}
36
+ {%- elif content is none or content is undefined %}
37
+ {{- '' }}
38
+ {%- else %}
39
+ {{- raise_exception('Unexpected content type.') }}
40
+ {%- endif %}
41
+ {%- endmacro %}
42
+ {%- if not messages %}
43
+ {{- raise_exception('No messages provided.') }}
44
+ {%- endif %}
45
+ {%- if tools and tools is iterable and tools is not mapping %}
46
+ {{- '<|im_start|>system\n' }}
47
+ {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
48
+ {%- for tool in tools %}
49
+ {{- "\n" }}
50
+ {{- tool | tojson }}
51
+ {%- endfor %}
52
+ {{- "\n</tools>" }}
53
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
54
+ {%- if messages[0].role == 'system' %}
55
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
56
+ {%- if content %}
57
+ {{- '\n\n' + content }}
58
+ {%- endif %}
59
+ {%- endif %}
60
+ {{- '<|im_end|>\n' }}
61
+ {%- else %}
62
+ {%- if messages[0].role == 'system' %}
63
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
64
+ {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
65
+ {%- endif %}
66
+ {%- endif %}
67
+ {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
68
+ {%- for message in messages[::-1] %}
69
+ {%- set index = (messages|length - 1) - loop.index0 %}
70
+ {%- if ns.multi_step_tool and message.role == "user" %}
71
+ {%- set content = render_content(message.content, false)|trim %}
72
+ {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
73
+ {%- set ns.multi_step_tool = false %}
74
+ {%- set ns.last_query_index = index %}
75
+ {%- endif %}
76
+ {%- endif %}
77
+ {%- endfor %}
78
+ {%- if ns.multi_step_tool %}
79
+ {{- raise_exception('No user query found in messages.') }}
80
+ {%- endif %}
81
+ {%- for message in messages %}
82
+ {%- set content = render_content(message.content, true)|trim %}
83
+ {%- if message.role == "system" %}
84
+ {%- if not loop.first %}
85
+ {{- raise_exception('System message must be at the beginning.') }}
86
+ {%- endif %}
87
+ {%- elif message.role == "user" %}
88
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
89
+ {%- elif message.role == "assistant" %}
90
+ {%- set reasoning_content = '' %}
91
+ {%- if message.reasoning_content is string %}
92
+ {%- set reasoning_content = message.reasoning_content %}
93
+ {%- else %}
94
+ {%- if '</think>' in content %}
95
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
96
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
97
+ {%- endif %}
98
+ {%- endif %}
99
+ {%- set reasoning_content = reasoning_content|trim %}
100
+ {%- if loop.index0 > ns.last_query_index %}
101
+ {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
102
+ {%- else %}
103
+ {{- '<|im_start|>' + message.role + '\n' + content }}
104
+ {%- endif %}
105
+ {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
106
+ {%- for tool_call in message.tool_calls %}
107
+ {%- if tool_call.function is defined %}
108
+ {%- set tool_call = tool_call.function %}
109
+ {%- endif %}
110
+ {%- if loop.first %}
111
+ {%- if content|trim %}
112
+ {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
113
+ {%- else %}
114
+ {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
115
+ {%- endif %}
116
+ {%- else %}
117
+ {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
118
+ {%- endif %}
119
+ {%- if tool_call.arguments is defined %}
120
+ {%- for args_name, args_value in tool_call.arguments|items %}
121
+ {{- '<parameter=' + args_name + '>\n' }}
122
+ {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
123
+ {{- args_value }}
124
+ {{- '\n</parameter>\n' }}
125
+ {%- endfor %}
126
+ {%- endif %}
127
+ {{- '</function>\n</tool_call>' }}
128
+ {%- endfor %}
129
+ {%- endif %}
130
+ {{- '<|im_end|>\n' }}
131
+ {%- elif message.role == "tool" %}
132
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
133
+ {{- '<|im_start|>user' }}
134
+ {%- endif %}
135
+ {{- '\n<tool_response>\n' }}
136
+ {{- content }}
137
+ {{- '\n</tool_response>' }}
138
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
139
+ {{- '<|im_end|>\n' }}
140
+ {%- elif loop.last %}
141
+ {{- '<|im_end|>\n' }}
142
+ {%- endif %}
143
+ {%- else %}
144
+ {{- raise_exception('Unexpected message role.') }}
145
+ {%- endif %}
146
+ {%- endfor %}
147
+ {%- if add_generation_prompt %}
148
+ {{- '<|im_start|>assistant\n' }}
149
+ {%- if enable_thinking is defined and enable_thinking is false %}
150
+ {{- '<think>\n\n</think>\n\n' }}
151
+ {%- else %}
152
+ {{- '<think>\n' }}
153
+ {%- endif %}
154
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,1243 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3_5ForConditionalGeneration"
4
+ ],
5
+ "image_token_id": 248056,
6
+ "model_type": "qwen3_5",
7
+ "quantization": {
8
+ "group_size": 64,
9
+ "bits": 4,
10
+ "mode": "affine",
11
+ "language_model.model.layers.0.linear_attn.in_proj_qkv": {
12
+ "group_size": 64,
13
+ "bits": 6,
14
+ "mode": "affine"
15
+ },
16
+ "language_model.model.layers.0.linear_attn.in_proj_z": {
17
+ "group_size": 64,
18
+ "bits": 6,
19
+ "mode": "affine"
20
+ },
21
+ "language_model.model.layers.0.linear_attn.out_proj": {
22
+ "group_size": 64,
23
+ "bits": 8,
24
+ "mode": "affine"
25
+ },
26
+ "language_model.model.layers.0.mlp.down_proj": {
27
+ "group_size": 64,
28
+ "bits": 6,
29
+ "mode": "affine"
30
+ },
31
+ "language_model.model.layers.1.linear_attn.in_proj_qkv": {
32
+ "group_size": 64,
33
+ "bits": 6,
34
+ "mode": "affine"
35
+ },
36
+ "language_model.model.layers.1.linear_attn.in_proj_z": {
37
+ "group_size": 64,
38
+ "bits": 6,
39
+ "mode": "affine"
40
+ },
41
+ "language_model.model.layers.1.linear_attn.out_proj": {
42
+ "group_size": 64,
43
+ "bits": 8,
44
+ "mode": "affine"
45
+ },
46
+ "language_model.model.layers.1.mlp.down_proj": {
47
+ "group_size": 64,
48
+ "bits": 6,
49
+ "mode": "affine"
50
+ },
51
+ "language_model.model.layers.2.linear_attn.in_proj_qkv": {
52
+ "group_size": 64,
53
+ "bits": 6,
54
+ "mode": "affine"
55
+ },
56
+ "language_model.model.layers.2.linear_attn.in_proj_z": {
57
+ "group_size": 64,
58
+ "bits": 6,
59
+ "mode": "affine"
60
+ },
61
+ "language_model.model.layers.2.linear_attn.out_proj": {
62
+ "group_size": 64,
63
+ "bits": 8,
64
+ "mode": "affine"
65
+ },
66
+ "language_model.model.layers.2.mlp.down_proj": {
67
+ "group_size": 64,
68
+ "bits": 6,
69
+ "mode": "affine"
70
+ },
71
+ "language_model.model.layers.3.self_attn.v_proj": {
72
+ "group_size": 64,
73
+ "bits": 6,
74
+ "mode": "affine"
75
+ },
76
+ "language_model.model.layers.3.mlp.down_proj": {
77
+ "group_size": 64,
78
+ "bits": 6,
79
+ "mode": "affine"
80
+ },
81
+ "language_model.model.layers.4.linear_attn.in_proj_qkv": {
82
+ "group_size": 64,
83
+ "bits": 6,
84
+ "mode": "affine"
85
+ },
86
+ "language_model.model.layers.4.linear_attn.in_proj_z": {
87
+ "group_size": 64,
88
+ "bits": 6,
89
+ "mode": "affine"
90
+ },
91
+ "language_model.model.layers.4.linear_attn.out_proj": {
92
+ "group_size": 64,
93
+ "bits": 8,
94
+ "mode": "affine"
95
+ },
96
+ "language_model.model.layers.4.mlp.down_proj": {
97
+ "group_size": 64,
98
+ "bits": 6,
99
+ "mode": "affine"
100
+ },
101
+ "language_model.model.layers.5.linear_attn.in_proj_qkv": {
102
+ "group_size": 64,
103
+ "bits": 6,
104
+ "mode": "affine"
105
+ },
106
+ "language_model.model.layers.5.linear_attn.in_proj_z": {
107
+ "group_size": 64,
108
+ "bits": 6,
109
+ "mode": "affine"
110
+ },
111
+ "language_model.model.layers.5.linear_attn.out_proj": {
112
+ "group_size": 64,
113
+ "bits": 8,
114
+ "mode": "affine"
115
+ },
116
+ "language_model.model.layers.5.mlp.down_proj": {
117
+ "group_size": 64,
118
+ "bits": 6,
119
+ "mode": "affine"
120
+ },
121
+ "language_model.model.layers.6.linear_attn.in_proj_qkv": {
122
+ "group_size": 64,
123
+ "bits": 6,
124
+ "mode": "affine"
125
+ },
126
+ "language_model.model.layers.6.linear_attn.in_proj_z": {
127
+ "group_size": 64,
128
+ "bits": 6,
129
+ "mode": "affine"
130
+ },
131
+ "language_model.model.layers.6.linear_attn.out_proj": {
132
+ "group_size": 64,
133
+ "bits": 8,
134
+ "mode": "affine"
135
+ },
136
+ "language_model.model.layers.6.mlp.down_proj": {
137
+ "group_size": 64,
138
+ "bits": 6,
139
+ "mode": "affine"
140
+ },
141
+ "language_model.model.layers.7.self_attn.v_proj": {
142
+ "group_size": 64,
143
+ "bits": 6,
144
+ "mode": "affine"
145
+ },
146
+ "language_model.model.layers.7.mlp.down_proj": {
147
+ "group_size": 64,
148
+ "bits": 6,
149
+ "mode": "affine"
150
+ },
151
+ "language_model.model.layers.8.linear_attn.in_proj_qkv": {
152
+ "group_size": 64,
153
+ "bits": 6,
154
+ "mode": "affine"
155
+ },
156
+ "language_model.model.layers.8.linear_attn.in_proj_z": {
157
+ "group_size": 64,
158
+ "bits": 6,
159
+ "mode": "affine"
160
+ },
161
+ "language_model.model.layers.8.linear_attn.out_proj": {
162
+ "group_size": 64,
163
+ "bits": 8,
164
+ "mode": "affine"
165
+ },
166
+ "language_model.model.layers.8.mlp.down_proj": {
167
+ "group_size": 64,
168
+ "bits": 6,
169
+ "mode": "affine"
170
+ },
171
+ "language_model.model.layers.9.linear_attn.in_proj_qkv": {
172
+ "group_size": 64,
173
+ "bits": 6,
174
+ "mode": "affine"
175
+ },
176
+ "language_model.model.layers.9.linear_attn.in_proj_z": {
177
+ "group_size": 64,
178
+ "bits": 6,
179
+ "mode": "affine"
180
+ },
181
+ "language_model.model.layers.9.linear_attn.out_proj": {
182
+ "group_size": 64,
183
+ "bits": 8,
184
+ "mode": "affine"
185
+ },
186
+ "language_model.model.layers.9.mlp.down_proj": {
187
+ "group_size": 64,
188
+ "bits": 6,
189
+ "mode": "affine"
190
+ },
191
+ "language_model.model.layers.10.linear_attn.in_proj_qkv": {
192
+ "group_size": 64,
193
+ "bits": 6,
194
+ "mode": "affine"
195
+ },
196
+ "language_model.model.layers.10.linear_attn.in_proj_z": {
197
+ "group_size": 64,
198
+ "bits": 6,
199
+ "mode": "affine"
200
+ },
201
+ "language_model.model.layers.10.linear_attn.out_proj": {
202
+ "group_size": 64,
203
+ "bits": 8,
204
+ "mode": "affine"
205
+ },
206
+ "language_model.model.layers.10.mlp.down_proj": {
207
+ "group_size": 64,
208
+ "bits": 6,
209
+ "mode": "affine"
210
+ },
211
+ "language_model.model.layers.11.self_attn.v_proj": {
212
+ "group_size": 64,
213
+ "bits": 6,
214
+ "mode": "affine"
215
+ },
216
+ "language_model.model.layers.11.mlp.down_proj": {
217
+ "group_size": 64,
218
+ "bits": 6,
219
+ "mode": "affine"
220
+ },
221
+ "language_model.model.layers.12.linear_attn.in_proj_qkv": {
222
+ "group_size": 64,
223
+ "bits": 6,
224
+ "mode": "affine"
225
+ },
226
+ "language_model.model.layers.12.linear_attn.in_proj_z": {
227
+ "group_size": 64,
228
+ "bits": 6,
229
+ "mode": "affine"
230
+ },
231
+ "language_model.model.layers.12.linear_attn.out_proj": {
232
+ "group_size": 64,
233
+ "bits": 8,
234
+ "mode": "affine"
235
+ },
236
+ "language_model.model.layers.12.mlp.down_proj": {
237
+ "group_size": 64,
238
+ "bits": 6,
239
+ "mode": "affine"
240
+ },
241
+ "language_model.model.layers.13.linear_attn.in_proj_qkv": {
242
+ "group_size": 64,
243
+ "bits": 6,
244
+ "mode": "affine"
245
+ },
246
+ "language_model.model.layers.13.linear_attn.in_proj_z": {
247
+ "group_size": 64,
248
+ "bits": 6,
249
+ "mode": "affine"
250
+ },
251
+ "language_model.model.layers.13.linear_attn.out_proj": {
252
+ "group_size": 64,
253
+ "bits": 8,
254
+ "mode": "affine"
255
+ },
256
+ "language_model.model.layers.13.mlp.down_proj": {
257
+ "group_size": 64,
258
+ "bits": 6,
259
+ "mode": "affine"
260
+ },
261
+ "language_model.model.layers.14.linear_attn.in_proj_qkv": {
262
+ "group_size": 64,
263
+ "bits": 6,
264
+ "mode": "affine"
265
+ },
266
+ "language_model.model.layers.14.linear_attn.in_proj_z": {
267
+ "group_size": 64,
268
+ "bits": 6,
269
+ "mode": "affine"
270
+ },
271
+ "language_model.model.layers.14.linear_attn.out_proj": {
272
+ "group_size": 64,
273
+ "bits": 8,
274
+ "mode": "affine"
275
+ },
276
+ "language_model.model.layers.14.mlp.down_proj": {
277
+ "group_size": 64,
278
+ "bits": 6,
279
+ "mode": "affine"
280
+ },
281
+ "language_model.model.layers.15.self_attn.v_proj": {
282
+ "group_size": 64,
283
+ "bits": 6,
284
+ "mode": "affine"
285
+ },
286
+ "language_model.model.layers.15.mlp.down_proj": {
287
+ "group_size": 64,
288
+ "bits": 6,
289
+ "mode": "affine"
290
+ },
291
+ "language_model.model.layers.16.linear_attn.in_proj_qkv": {
292
+ "group_size": 64,
293
+ "bits": 6,
294
+ "mode": "affine"
295
+ },
296
+ "language_model.model.layers.16.linear_attn.in_proj_z": {
297
+ "group_size": 64,
298
+ "bits": 6,
299
+ "mode": "affine"
300
+ },
301
+ "language_model.model.layers.16.linear_attn.out_proj": {
302
+ "group_size": 64,
303
+ "bits": 8,
304
+ "mode": "affine"
305
+ },
306
+ "language_model.model.layers.16.mlp.down_proj": {
307
+ "group_size": 64,
308
+ "bits": 6,
309
+ "mode": "affine"
310
+ },
311
+ "language_model.model.layers.17.linear_attn.in_proj_qkv": {
312
+ "group_size": 64,
313
+ "bits": 6,
314
+ "mode": "affine"
315
+ },
316
+ "language_model.model.layers.17.linear_attn.in_proj_z": {
317
+ "group_size": 64,
318
+ "bits": 6,
319
+ "mode": "affine"
320
+ },
321
+ "language_model.model.layers.17.linear_attn.out_proj": {
322
+ "group_size": 64,
323
+ "bits": 8,
324
+ "mode": "affine"
325
+ },
326
+ "language_model.model.layers.17.mlp.down_proj": {
327
+ "group_size": 64,
328
+ "bits": 6,
329
+ "mode": "affine"
330
+ },
331
+ "language_model.model.layers.18.linear_attn.in_proj_qkv": {
332
+ "group_size": 64,
333
+ "bits": 6,
334
+ "mode": "affine"
335
+ },
336
+ "language_model.model.layers.18.linear_attn.in_proj_z": {
337
+ "group_size": 64,
338
+ "bits": 6,
339
+ "mode": "affine"
340
+ },
341
+ "language_model.model.layers.18.linear_attn.out_proj": {
342
+ "group_size": 64,
343
+ "bits": 8,
344
+ "mode": "affine"
345
+ },
346
+ "language_model.model.layers.18.mlp.down_proj": {
347
+ "group_size": 64,
348
+ "bits": 6,
349
+ "mode": "affine"
350
+ },
351
+ "language_model.model.layers.19.self_attn.v_proj": {
352
+ "group_size": 64,
353
+ "bits": 6,
354
+ "mode": "affine"
355
+ },
356
+ "language_model.model.layers.19.mlp.down_proj": {
357
+ "group_size": 64,
358
+ "bits": 6,
359
+ "mode": "affine"
360
+ },
361
+ "language_model.model.layers.20.linear_attn.in_proj_qkv": {
362
+ "group_size": 64,
363
+ "bits": 6,
364
+ "mode": "affine"
365
+ },
366
+ "language_model.model.layers.20.linear_attn.in_proj_z": {
367
+ "group_size": 64,
368
+ "bits": 6,
369
+ "mode": "affine"
370
+ },
371
+ "language_model.model.layers.20.linear_attn.out_proj": {
372
+ "group_size": 64,
373
+ "bits": 8,
374
+ "mode": "affine"
375
+ },
376
+ "language_model.model.layers.20.mlp.down_proj": {
377
+ "group_size": 64,
378
+ "bits": 6,
379
+ "mode": "affine"
380
+ },
381
+ "language_model.model.layers.21.linear_attn.in_proj_qkv": {
382
+ "group_size": 64,
383
+ "bits": 6,
384
+ "mode": "affine"
385
+ },
386
+ "language_model.model.layers.21.linear_attn.in_proj_z": {
387
+ "group_size": 64,
388
+ "bits": 6,
389
+ "mode": "affine"
390
+ },
391
+ "language_model.model.layers.21.linear_attn.out_proj": {
392
+ "group_size": 64,
393
+ "bits": 8,
394
+ "mode": "affine"
395
+ },
396
+ "language_model.model.layers.21.mlp.down_proj": {
397
+ "group_size": 64,
398
+ "bits": 6,
399
+ "mode": "affine"
400
+ },
401
+ "language_model.model.layers.22.linear_attn.in_proj_qkv": {
402
+ "group_size": 64,
403
+ "bits": 6,
404
+ "mode": "affine"
405
+ },
406
+ "language_model.model.layers.22.linear_attn.in_proj_z": {
407
+ "group_size": 64,
408
+ "bits": 6,
409
+ "mode": "affine"
410
+ },
411
+ "language_model.model.layers.22.linear_attn.out_proj": {
412
+ "group_size": 64,
413
+ "bits": 8,
414
+ "mode": "affine"
415
+ },
416
+ "language_model.model.layers.22.mlp.down_proj": {
417
+ "group_size": 64,
418
+ "bits": 6,
419
+ "mode": "affine"
420
+ },
421
+ "language_model.model.layers.23.self_attn.v_proj": {
422
+ "group_size": 64,
423
+ "bits": 6,
424
+ "mode": "affine"
425
+ },
426
+ "language_model.model.layers.23.mlp.down_proj": {
427
+ "group_size": 64,
428
+ "bits": 6,
429
+ "mode": "affine"
430
+ },
431
+ "language_model.model.layers.24.linear_attn.in_proj_qkv": {
432
+ "group_size": 64,
433
+ "bits": 6,
434
+ "mode": "affine"
435
+ },
436
+ "language_model.model.layers.24.linear_attn.in_proj_z": {
437
+ "group_size": 64,
438
+ "bits": 6,
439
+ "mode": "affine"
440
+ },
441
+ "language_model.model.layers.24.linear_attn.out_proj": {
442
+ "group_size": 64,
443
+ "bits": 8,
444
+ "mode": "affine"
445
+ },
446
+ "language_model.model.layers.24.mlp.down_proj": {
447
+ "group_size": 64,
448
+ "bits": 6,
449
+ "mode": "affine"
450
+ },
451
+ "language_model.model.layers.25.linear_attn.in_proj_qkv": {
452
+ "group_size": 64,
453
+ "bits": 6,
454
+ "mode": "affine"
455
+ },
456
+ "language_model.model.layers.25.linear_attn.in_proj_z": {
457
+ "group_size": 64,
458
+ "bits": 6,
459
+ "mode": "affine"
460
+ },
461
+ "language_model.model.layers.25.linear_attn.out_proj": {
462
+ "group_size": 64,
463
+ "bits": 8,
464
+ "mode": "affine"
465
+ },
466
+ "language_model.model.layers.25.mlp.down_proj": {
467
+ "group_size": 64,
468
+ "bits": 6,
469
+ "mode": "affine"
470
+ },
471
+ "language_model.model.layers.26.linear_attn.in_proj_qkv": {
472
+ "group_size": 64,
473
+ "bits": 6,
474
+ "mode": "affine"
475
+ },
476
+ "language_model.model.layers.26.linear_attn.in_proj_z": {
477
+ "group_size": 64,
478
+ "bits": 6,
479
+ "mode": "affine"
480
+ },
481
+ "language_model.model.layers.26.linear_attn.out_proj": {
482
+ "group_size": 64,
483
+ "bits": 8,
484
+ "mode": "affine"
485
+ },
486
+ "language_model.model.layers.26.mlp.down_proj": {
487
+ "group_size": 64,
488
+ "bits": 6,
489
+ "mode": "affine"
490
+ },
491
+ "language_model.model.layers.27.self_attn.v_proj": {
492
+ "group_size": 64,
493
+ "bits": 6,
494
+ "mode": "affine"
495
+ },
496
+ "language_model.model.layers.27.mlp.down_proj": {
497
+ "group_size": 64,
498
+ "bits": 6,
499
+ "mode": "affine"
500
+ },
501
+ "language_model.model.layers.28.linear_attn.in_proj_qkv": {
502
+ "group_size": 64,
503
+ "bits": 6,
504
+ "mode": "affine"
505
+ },
506
+ "language_model.model.layers.28.linear_attn.in_proj_z": {
507
+ "group_size": 64,
508
+ "bits": 6,
509
+ "mode": "affine"
510
+ },
511
+ "language_model.model.layers.28.linear_attn.out_proj": {
512
+ "group_size": 64,
513
+ "bits": 8,
514
+ "mode": "affine"
515
+ },
516
+ "language_model.model.layers.28.mlp.down_proj": {
517
+ "group_size": 64,
518
+ "bits": 6,
519
+ "mode": "affine"
520
+ },
521
+ "language_model.model.layers.29.linear_attn.in_proj_qkv": {
522
+ "group_size": 64,
523
+ "bits": 6,
524
+ "mode": "affine"
525
+ },
526
+ "language_model.model.layers.29.linear_attn.in_proj_z": {
527
+ "group_size": 64,
528
+ "bits": 6,
529
+ "mode": "affine"
530
+ },
531
+ "language_model.model.layers.29.linear_attn.out_proj": {
532
+ "group_size": 64,
533
+ "bits": 8,
534
+ "mode": "affine"
535
+ },
536
+ "language_model.model.layers.29.mlp.down_proj": {
537
+ "group_size": 64,
538
+ "bits": 6,
539
+ "mode": "affine"
540
+ },
541
+ "language_model.model.layers.30.linear_attn.in_proj_qkv": {
542
+ "group_size": 64,
543
+ "bits": 6,
544
+ "mode": "affine"
545
+ },
546
+ "language_model.model.layers.30.linear_attn.in_proj_z": {
547
+ "group_size": 64,
548
+ "bits": 6,
549
+ "mode": "affine"
550
+ },
551
+ "language_model.model.layers.30.linear_attn.out_proj": {
552
+ "group_size": 64,
553
+ "bits": 8,
554
+ "mode": "affine"
555
+ },
556
+ "language_model.model.layers.30.mlp.down_proj": {
557
+ "group_size": 64,
558
+ "bits": 6,
559
+ "mode": "affine"
560
+ },
561
+ "language_model.model.layers.31.self_attn.v_proj": {
562
+ "group_size": 64,
563
+ "bits": 6,
564
+ "mode": "affine"
565
+ },
566
+ "language_model.model.layers.31.mlp.down_proj": {
567
+ "group_size": 64,
568
+ "bits": 6,
569
+ "mode": "affine"
570
+ },
571
+ "language_model.lm_head": {
572
+ "group_size": 64,
573
+ "bits": 6,
574
+ "mode": "affine"
575
+ }
576
+ },
577
+ "quantization_config": {
578
+ "group_size": 64,
579
+ "bits": 4,
580
+ "mode": "affine",
581
+ "language_model.model.layers.0.linear_attn.in_proj_qkv": {
582
+ "group_size": 64,
583
+ "bits": 6,
584
+ "mode": "affine"
585
+ },
586
+ "language_model.model.layers.0.linear_attn.in_proj_z": {
587
+ "group_size": 64,
588
+ "bits": 6,
589
+ "mode": "affine"
590
+ },
591
+ "language_model.model.layers.0.linear_attn.out_proj": {
592
+ "group_size": 64,
593
+ "bits": 8,
594
+ "mode": "affine"
595
+ },
596
+ "language_model.model.layers.0.mlp.down_proj": {
597
+ "group_size": 64,
598
+ "bits": 6,
599
+ "mode": "affine"
600
+ },
601
+ "language_model.model.layers.1.linear_attn.in_proj_qkv": {
602
+ "group_size": 64,
603
+ "bits": 6,
604
+ "mode": "affine"
605
+ },
606
+ "language_model.model.layers.1.linear_attn.in_proj_z": {
607
+ "group_size": 64,
608
+ "bits": 6,
609
+ "mode": "affine"
610
+ },
611
+ "language_model.model.layers.1.linear_attn.out_proj": {
612
+ "group_size": 64,
613
+ "bits": 8,
614
+ "mode": "affine"
615
+ },
616
+ "language_model.model.layers.1.mlp.down_proj": {
617
+ "group_size": 64,
618
+ "bits": 6,
619
+ "mode": "affine"
620
+ },
621
+ "language_model.model.layers.2.linear_attn.in_proj_qkv": {
622
+ "group_size": 64,
623
+ "bits": 6,
624
+ "mode": "affine"
625
+ },
626
+ "language_model.model.layers.2.linear_attn.in_proj_z": {
627
+ "group_size": 64,
628
+ "bits": 6,
629
+ "mode": "affine"
630
+ },
631
+ "language_model.model.layers.2.linear_attn.out_proj": {
632
+ "group_size": 64,
633
+ "bits": 8,
634
+ "mode": "affine"
635
+ },
636
+ "language_model.model.layers.2.mlp.down_proj": {
637
+ "group_size": 64,
638
+ "bits": 6,
639
+ "mode": "affine"
640
+ },
641
+ "language_model.model.layers.3.self_attn.v_proj": {
642
+ "group_size": 64,
643
+ "bits": 6,
644
+ "mode": "affine"
645
+ },
646
+ "language_model.model.layers.3.mlp.down_proj": {
647
+ "group_size": 64,
648
+ "bits": 6,
649
+ "mode": "affine"
650
+ },
651
+ "language_model.model.layers.4.linear_attn.in_proj_qkv": {
652
+ "group_size": 64,
653
+ "bits": 6,
654
+ "mode": "affine"
655
+ },
656
+ "language_model.model.layers.4.linear_attn.in_proj_z": {
657
+ "group_size": 64,
658
+ "bits": 6,
659
+ "mode": "affine"
660
+ },
661
+ "language_model.model.layers.4.linear_attn.out_proj": {
662
+ "group_size": 64,
663
+ "bits": 8,
664
+ "mode": "affine"
665
+ },
666
+ "language_model.model.layers.4.mlp.down_proj": {
667
+ "group_size": 64,
668
+ "bits": 6,
669
+ "mode": "affine"
670
+ },
671
+ "language_model.model.layers.5.linear_attn.in_proj_qkv": {
672
+ "group_size": 64,
673
+ "bits": 6,
674
+ "mode": "affine"
675
+ },
676
+ "language_model.model.layers.5.linear_attn.in_proj_z": {
677
+ "group_size": 64,
678
+ "bits": 6,
679
+ "mode": "affine"
680
+ },
681
+ "language_model.model.layers.5.linear_attn.out_proj": {
682
+ "group_size": 64,
683
+ "bits": 8,
684
+ "mode": "affine"
685
+ },
686
+ "language_model.model.layers.5.mlp.down_proj": {
687
+ "group_size": 64,
688
+ "bits": 6,
689
+ "mode": "affine"
690
+ },
691
+ "language_model.model.layers.6.linear_attn.in_proj_qkv": {
692
+ "group_size": 64,
693
+ "bits": 6,
694
+ "mode": "affine"
695
+ },
696
+ "language_model.model.layers.6.linear_attn.in_proj_z": {
697
+ "group_size": 64,
698
+ "bits": 6,
699
+ "mode": "affine"
700
+ },
701
+ "language_model.model.layers.6.linear_attn.out_proj": {
702
+ "group_size": 64,
703
+ "bits": 8,
704
+ "mode": "affine"
705
+ },
706
+ "language_model.model.layers.6.mlp.down_proj": {
707
+ "group_size": 64,
708
+ "bits": 6,
709
+ "mode": "affine"
710
+ },
711
+ "language_model.model.layers.7.self_attn.v_proj": {
712
+ "group_size": 64,
713
+ "bits": 6,
714
+ "mode": "affine"
715
+ },
716
+ "language_model.model.layers.7.mlp.down_proj": {
717
+ "group_size": 64,
718
+ "bits": 6,
719
+ "mode": "affine"
720
+ },
721
+ "language_model.model.layers.8.linear_attn.in_proj_qkv": {
722
+ "group_size": 64,
723
+ "bits": 6,
724
+ "mode": "affine"
725
+ },
726
+ "language_model.model.layers.8.linear_attn.in_proj_z": {
727
+ "group_size": 64,
728
+ "bits": 6,
729
+ "mode": "affine"
730
+ },
731
+ "language_model.model.layers.8.linear_attn.out_proj": {
732
+ "group_size": 64,
733
+ "bits": 8,
734
+ "mode": "affine"
735
+ },
736
+ "language_model.model.layers.8.mlp.down_proj": {
737
+ "group_size": 64,
738
+ "bits": 6,
739
+ "mode": "affine"
740
+ },
741
+ "language_model.model.layers.9.linear_attn.in_proj_qkv": {
742
+ "group_size": 64,
743
+ "bits": 6,
744
+ "mode": "affine"
745
+ },
746
+ "language_model.model.layers.9.linear_attn.in_proj_z": {
747
+ "group_size": 64,
748
+ "bits": 6,
749
+ "mode": "affine"
750
+ },
751
+ "language_model.model.layers.9.linear_attn.out_proj": {
752
+ "group_size": 64,
753
+ "bits": 8,
754
+ "mode": "affine"
755
+ },
756
+ "language_model.model.layers.9.mlp.down_proj": {
757
+ "group_size": 64,
758
+ "bits": 6,
759
+ "mode": "affine"
760
+ },
761
+ "language_model.model.layers.10.linear_attn.in_proj_qkv": {
762
+ "group_size": 64,
763
+ "bits": 6,
764
+ "mode": "affine"
765
+ },
766
+ "language_model.model.layers.10.linear_attn.in_proj_z": {
767
+ "group_size": 64,
768
+ "bits": 6,
769
+ "mode": "affine"
770
+ },
771
+ "language_model.model.layers.10.linear_attn.out_proj": {
772
+ "group_size": 64,
773
+ "bits": 8,
774
+ "mode": "affine"
775
+ },
776
+ "language_model.model.layers.10.mlp.down_proj": {
777
+ "group_size": 64,
778
+ "bits": 6,
779
+ "mode": "affine"
780
+ },
781
+ "language_model.model.layers.11.self_attn.v_proj": {
782
+ "group_size": 64,
783
+ "bits": 6,
784
+ "mode": "affine"
785
+ },
786
+ "language_model.model.layers.11.mlp.down_proj": {
787
+ "group_size": 64,
788
+ "bits": 6,
789
+ "mode": "affine"
790
+ },
791
+ "language_model.model.layers.12.linear_attn.in_proj_qkv": {
792
+ "group_size": 64,
793
+ "bits": 6,
794
+ "mode": "affine"
795
+ },
796
+ "language_model.model.layers.12.linear_attn.in_proj_z": {
797
+ "group_size": 64,
798
+ "bits": 6,
799
+ "mode": "affine"
800
+ },
801
+ "language_model.model.layers.12.linear_attn.out_proj": {
802
+ "group_size": 64,
803
+ "bits": 8,
804
+ "mode": "affine"
805
+ },
806
+ "language_model.model.layers.12.mlp.down_proj": {
807
+ "group_size": 64,
808
+ "bits": 6,
809
+ "mode": "affine"
810
+ },
811
+ "language_model.model.layers.13.linear_attn.in_proj_qkv": {
812
+ "group_size": 64,
813
+ "bits": 6,
814
+ "mode": "affine"
815
+ },
816
+ "language_model.model.layers.13.linear_attn.in_proj_z": {
817
+ "group_size": 64,
818
+ "bits": 6,
819
+ "mode": "affine"
820
+ },
821
+ "language_model.model.layers.13.linear_attn.out_proj": {
822
+ "group_size": 64,
823
+ "bits": 8,
824
+ "mode": "affine"
825
+ },
826
+ "language_model.model.layers.13.mlp.down_proj": {
827
+ "group_size": 64,
828
+ "bits": 6,
829
+ "mode": "affine"
830
+ },
831
+ "language_model.model.layers.14.linear_attn.in_proj_qkv": {
832
+ "group_size": 64,
833
+ "bits": 6,
834
+ "mode": "affine"
835
+ },
836
+ "language_model.model.layers.14.linear_attn.in_proj_z": {
837
+ "group_size": 64,
838
+ "bits": 6,
839
+ "mode": "affine"
840
+ },
841
+ "language_model.model.layers.14.linear_attn.out_proj": {
842
+ "group_size": 64,
843
+ "bits": 8,
844
+ "mode": "affine"
845
+ },
846
+ "language_model.model.layers.14.mlp.down_proj": {
847
+ "group_size": 64,
848
+ "bits": 6,
849
+ "mode": "affine"
850
+ },
851
+ "language_model.model.layers.15.self_attn.v_proj": {
852
+ "group_size": 64,
853
+ "bits": 6,
854
+ "mode": "affine"
855
+ },
856
+ "language_model.model.layers.15.mlp.down_proj": {
857
+ "group_size": 64,
858
+ "bits": 6,
859
+ "mode": "affine"
860
+ },
861
+ "language_model.model.layers.16.linear_attn.in_proj_qkv": {
862
+ "group_size": 64,
863
+ "bits": 6,
864
+ "mode": "affine"
865
+ },
866
+ "language_model.model.layers.16.linear_attn.in_proj_z": {
867
+ "group_size": 64,
868
+ "bits": 6,
869
+ "mode": "affine"
870
+ },
871
+ "language_model.model.layers.16.linear_attn.out_proj": {
872
+ "group_size": 64,
873
+ "bits": 8,
874
+ "mode": "affine"
875
+ },
876
+ "language_model.model.layers.16.mlp.down_proj": {
877
+ "group_size": 64,
878
+ "bits": 6,
879
+ "mode": "affine"
880
+ },
881
+ "language_model.model.layers.17.linear_attn.in_proj_qkv": {
882
+ "group_size": 64,
883
+ "bits": 6,
884
+ "mode": "affine"
885
+ },
886
+ "language_model.model.layers.17.linear_attn.in_proj_z": {
887
+ "group_size": 64,
888
+ "bits": 6,
889
+ "mode": "affine"
890
+ },
891
+ "language_model.model.layers.17.linear_attn.out_proj": {
892
+ "group_size": 64,
893
+ "bits": 8,
894
+ "mode": "affine"
895
+ },
896
+ "language_model.model.layers.17.mlp.down_proj": {
897
+ "group_size": 64,
898
+ "bits": 6,
899
+ "mode": "affine"
900
+ },
901
+ "language_model.model.layers.18.linear_attn.in_proj_qkv": {
902
+ "group_size": 64,
903
+ "bits": 6,
904
+ "mode": "affine"
905
+ },
906
+ "language_model.model.layers.18.linear_attn.in_proj_z": {
907
+ "group_size": 64,
908
+ "bits": 6,
909
+ "mode": "affine"
910
+ },
911
+ "language_model.model.layers.18.linear_attn.out_proj": {
912
+ "group_size": 64,
913
+ "bits": 8,
914
+ "mode": "affine"
915
+ },
916
+ "language_model.model.layers.18.mlp.down_proj": {
917
+ "group_size": 64,
918
+ "bits": 6,
919
+ "mode": "affine"
920
+ },
921
+ "language_model.model.layers.19.self_attn.v_proj": {
922
+ "group_size": 64,
923
+ "bits": 6,
924
+ "mode": "affine"
925
+ },
926
+ "language_model.model.layers.19.mlp.down_proj": {
927
+ "group_size": 64,
928
+ "bits": 6,
929
+ "mode": "affine"
930
+ },
931
+ "language_model.model.layers.20.linear_attn.in_proj_qkv": {
932
+ "group_size": 64,
933
+ "bits": 6,
934
+ "mode": "affine"
935
+ },
936
+ "language_model.model.layers.20.linear_attn.in_proj_z": {
937
+ "group_size": 64,
938
+ "bits": 6,
939
+ "mode": "affine"
940
+ },
941
+ "language_model.model.layers.20.linear_attn.out_proj": {
942
+ "group_size": 64,
943
+ "bits": 8,
944
+ "mode": "affine"
945
+ },
946
+ "language_model.model.layers.20.mlp.down_proj": {
947
+ "group_size": 64,
948
+ "bits": 6,
949
+ "mode": "affine"
950
+ },
951
+ "language_model.model.layers.21.linear_attn.in_proj_qkv": {
952
+ "group_size": 64,
953
+ "bits": 6,
954
+ "mode": "affine"
955
+ },
956
+ "language_model.model.layers.21.linear_attn.in_proj_z": {
957
+ "group_size": 64,
958
+ "bits": 6,
959
+ "mode": "affine"
960
+ },
961
+ "language_model.model.layers.21.linear_attn.out_proj": {
962
+ "group_size": 64,
963
+ "bits": 8,
964
+ "mode": "affine"
965
+ },
966
+ "language_model.model.layers.21.mlp.down_proj": {
967
+ "group_size": 64,
968
+ "bits": 6,
969
+ "mode": "affine"
970
+ },
971
+ "language_model.model.layers.22.linear_attn.in_proj_qkv": {
972
+ "group_size": 64,
973
+ "bits": 6,
974
+ "mode": "affine"
975
+ },
976
+ "language_model.model.layers.22.linear_attn.in_proj_z": {
977
+ "group_size": 64,
978
+ "bits": 6,
979
+ "mode": "affine"
980
+ },
981
+ "language_model.model.layers.22.linear_attn.out_proj": {
982
+ "group_size": 64,
983
+ "bits": 8,
984
+ "mode": "affine"
985
+ },
986
+ "language_model.model.layers.22.mlp.down_proj": {
987
+ "group_size": 64,
988
+ "bits": 6,
989
+ "mode": "affine"
990
+ },
991
+ "language_model.model.layers.23.self_attn.v_proj": {
992
+ "group_size": 64,
993
+ "bits": 6,
994
+ "mode": "affine"
995
+ },
996
+ "language_model.model.layers.23.mlp.down_proj": {
997
+ "group_size": 64,
998
+ "bits": 6,
999
+ "mode": "affine"
1000
+ },
1001
+ "language_model.model.layers.24.linear_attn.in_proj_qkv": {
1002
+ "group_size": 64,
1003
+ "bits": 6,
1004
+ "mode": "affine"
1005
+ },
1006
+ "language_model.model.layers.24.linear_attn.in_proj_z": {
1007
+ "group_size": 64,
1008
+ "bits": 6,
1009
+ "mode": "affine"
1010
+ },
1011
+ "language_model.model.layers.24.linear_attn.out_proj": {
1012
+ "group_size": 64,
1013
+ "bits": 8,
1014
+ "mode": "affine"
1015
+ },
1016
+ "language_model.model.layers.24.mlp.down_proj": {
1017
+ "group_size": 64,
1018
+ "bits": 6,
1019
+ "mode": "affine"
1020
+ },
1021
+ "language_model.model.layers.25.linear_attn.in_proj_qkv": {
1022
+ "group_size": 64,
1023
+ "bits": 6,
1024
+ "mode": "affine"
1025
+ },
1026
+ "language_model.model.layers.25.linear_attn.in_proj_z": {
1027
+ "group_size": 64,
1028
+ "bits": 6,
1029
+ "mode": "affine"
1030
+ },
1031
+ "language_model.model.layers.25.linear_attn.out_proj": {
1032
+ "group_size": 64,
1033
+ "bits": 8,
1034
+ "mode": "affine"
1035
+ },
1036
+ "language_model.model.layers.25.mlp.down_proj": {
1037
+ "group_size": 64,
1038
+ "bits": 6,
1039
+ "mode": "affine"
1040
+ },
1041
+ "language_model.model.layers.26.linear_attn.in_proj_qkv": {
1042
+ "group_size": 64,
1043
+ "bits": 6,
1044
+ "mode": "affine"
1045
+ },
1046
+ "language_model.model.layers.26.linear_attn.in_proj_z": {
1047
+ "group_size": 64,
1048
+ "bits": 6,
1049
+ "mode": "affine"
1050
+ },
1051
+ "language_model.model.layers.26.linear_attn.out_proj": {
1052
+ "group_size": 64,
1053
+ "bits": 8,
1054
+ "mode": "affine"
1055
+ },
1056
+ "language_model.model.layers.26.mlp.down_proj": {
1057
+ "group_size": 64,
1058
+ "bits": 6,
1059
+ "mode": "affine"
1060
+ },
1061
+ "language_model.model.layers.27.self_attn.v_proj": {
1062
+ "group_size": 64,
1063
+ "bits": 6,
1064
+ "mode": "affine"
1065
+ },
1066
+ "language_model.model.layers.27.mlp.down_proj": {
1067
+ "group_size": 64,
1068
+ "bits": 6,
1069
+ "mode": "affine"
1070
+ },
1071
+ "language_model.model.layers.28.linear_attn.in_proj_qkv": {
1072
+ "group_size": 64,
1073
+ "bits": 6,
1074
+ "mode": "affine"
1075
+ },
1076
+ "language_model.model.layers.28.linear_attn.in_proj_z": {
1077
+ "group_size": 64,
1078
+ "bits": 6,
1079
+ "mode": "affine"
1080
+ },
1081
+ "language_model.model.layers.28.linear_attn.out_proj": {
1082
+ "group_size": 64,
1083
+ "bits": 8,
1084
+ "mode": "affine"
1085
+ },
1086
+ "language_model.model.layers.28.mlp.down_proj": {
1087
+ "group_size": 64,
1088
+ "bits": 6,
1089
+ "mode": "affine"
1090
+ },
1091
+ "language_model.model.layers.29.linear_attn.in_proj_qkv": {
1092
+ "group_size": 64,
1093
+ "bits": 6,
1094
+ "mode": "affine"
1095
+ },
1096
+ "language_model.model.layers.29.linear_attn.in_proj_z": {
1097
+ "group_size": 64,
1098
+ "bits": 6,
1099
+ "mode": "affine"
1100
+ },
1101
+ "language_model.model.layers.29.linear_attn.out_proj": {
1102
+ "group_size": 64,
1103
+ "bits": 8,
1104
+ "mode": "affine"
1105
+ },
1106
+ "language_model.model.layers.29.mlp.down_proj": {
1107
+ "group_size": 64,
1108
+ "bits": 6,
1109
+ "mode": "affine"
1110
+ },
1111
+ "language_model.model.layers.30.linear_attn.in_proj_qkv": {
1112
+ "group_size": 64,
1113
+ "bits": 6,
1114
+ "mode": "affine"
1115
+ },
1116
+ "language_model.model.layers.30.linear_attn.in_proj_z": {
1117
+ "group_size": 64,
1118
+ "bits": 6,
1119
+ "mode": "affine"
1120
+ },
1121
+ "language_model.model.layers.30.linear_attn.out_proj": {
1122
+ "group_size": 64,
1123
+ "bits": 8,
1124
+ "mode": "affine"
1125
+ },
1126
+ "language_model.model.layers.30.mlp.down_proj": {
1127
+ "group_size": 64,
1128
+ "bits": 6,
1129
+ "mode": "affine"
1130
+ },
1131
+ "language_model.model.layers.31.self_attn.v_proj": {
1132
+ "group_size": 64,
1133
+ "bits": 6,
1134
+ "mode": "affine"
1135
+ },
1136
+ "language_model.model.layers.31.mlp.down_proj": {
1137
+ "group_size": 64,
1138
+ "bits": 6,
1139
+ "mode": "affine"
1140
+ },
1141
+ "language_model.lm_head": {
1142
+ "group_size": 64,
1143
+ "bits": 6,
1144
+ "mode": "affine"
1145
+ }
1146
+ },
1147
+ "text_config": {
1148
+ "attention_bias": false,
1149
+ "attention_dropout": 0.0,
1150
+ "attn_output_gate": true,
1151
+ "dtype": "bfloat16",
1152
+ "eos_token_id": 248044,
1153
+ "full_attention_interval": 4,
1154
+ "head_dim": 256,
1155
+ "hidden_act": "silu",
1156
+ "hidden_size": 4096,
1157
+ "initializer_range": 0.02,
1158
+ "intermediate_size": 12288,
1159
+ "layer_types": [
1160
+ "linear_attention",
1161
+ "linear_attention",
1162
+ "linear_attention",
1163
+ "full_attention",
1164
+ "linear_attention",
1165
+ "linear_attention",
1166
+ "linear_attention",
1167
+ "full_attention",
1168
+ "linear_attention",
1169
+ "linear_attention",
1170
+ "linear_attention",
1171
+ "full_attention",
1172
+ "linear_attention",
1173
+ "linear_attention",
1174
+ "linear_attention",
1175
+ "full_attention",
1176
+ "linear_attention",
1177
+ "linear_attention",
1178
+ "linear_attention",
1179
+ "full_attention",
1180
+ "linear_attention",
1181
+ "linear_attention",
1182
+ "linear_attention",
1183
+ "full_attention",
1184
+ "linear_attention",
1185
+ "linear_attention",
1186
+ "linear_attention",
1187
+ "full_attention",
1188
+ "linear_attention",
1189
+ "linear_attention",
1190
+ "linear_attention",
1191
+ "full_attention"
1192
+ ],
1193
+ "linear_conv_kernel_dim": 4,
1194
+ "linear_key_head_dim": 128,
1195
+ "linear_num_key_heads": 16,
1196
+ "linear_num_value_heads": 32,
1197
+ "linear_value_head_dim": 128,
1198
+ "max_position_embeddings": 262144,
1199
+ "mlp_only_layers": [],
1200
+ "model_type": "qwen3_5_text",
1201
+ "mtp_num_hidden_layers": 1,
1202
+ "mtp_use_dedicated_embeddings": false,
1203
+ "num_attention_heads": 16,
1204
+ "num_hidden_layers": 32,
1205
+ "num_key_value_heads": 4,
1206
+ "rms_norm_eps": 1e-06,
1207
+ "use_cache": true,
1208
+ "vocab_size": 248320,
1209
+ "mamba_ssm_dtype": "float32",
1210
+ "rope_parameters": {
1211
+ "mrope_interleaved": true,
1212
+ "mrope_section": [
1213
+ 11,
1214
+ 11,
1215
+ 10
1216
+ ],
1217
+ "rope_type": "default",
1218
+ "rope_theta": 10000000,
1219
+ "partial_rotary_factor": 0.25
1220
+ }
1221
+ },
1222
+ "tie_word_embeddings": false,
1223
+ "transformers_version": "4.57.0.dev0",
1224
+ "video_token_id": 248057,
1225
+ "vision_config": {
1226
+ "deepstack_visual_indexes": [],
1227
+ "depth": 27,
1228
+ "hidden_act": "gelu_pytorch_tanh",
1229
+ "hidden_size": 1152,
1230
+ "in_channels": 3,
1231
+ "initializer_range": 0.02,
1232
+ "intermediate_size": 4304,
1233
+ "model_type": "qwen3_5",
1234
+ "num_heads": 16,
1235
+ "num_position_embeddings": 2304,
1236
+ "out_hidden_size": 4096,
1237
+ "patch_size": 16,
1238
+ "spatial_merge_size": 2,
1239
+ "temporal_patch_size": 2
1240
+ },
1241
+ "vision_end_token_id": 248054,
1242
+ "vision_start_token_id": 248053
1243
+ }
model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99a82b139ea8348badc412acb1b3766e769a9dafcb353b0c9caac2da3ea9f89d
3
+ size 5341640539
model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:defe883970f4d2eb6781df0fb6af78314211edbf89cce41fa70de5a4dafe98c0
3
+ size 1786249784
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
preprocessor_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "size": {
3
+ "longest_edge": 16777216,
4
+ "shortest_edge": 65536
5
+ },
6
+ "patch_size": 16,
7
+ "temporal_patch_size": 2,
8
+ "merge_size": 2,
9
+ "image_mean": [
10
+ 0.5,
11
+ 0.5,
12
+ 0.5
13
+ ],
14
+ "image_std": [
15
+ 0.5,
16
+ 0.5,
17
+ 0.5
18
+ ],
19
+ "processor_class": "Qwen3VLProcessor",
20
+ "image_processor_type": "Qwen2VLImageProcessorFast"
21
+ }
processor_config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "do_convert_rgb": true,
4
+ "do_normalize": true,
5
+ "do_rescale": true,
6
+ "image_mean": [
7
+ 0.5,
8
+ 0.5,
9
+ 0.5
10
+ ],
11
+ "image_processor_type": "Qwen3VLImageProcessor",
12
+ "image_std": [
13
+ 0.5,
14
+ 0.5,
15
+ 0.5
16
+ ],
17
+ "max_pixels": 16777216,
18
+ "merge_size": 2,
19
+ "min_pixels": 65536,
20
+ "patch_size": 16,
21
+ "rescale_factor": 0.00392156862745098,
22
+ "temporal_patch_size": 2
23
+ },
24
+ "processor_class": "Qwen3VLProcessor",
25
+ "video_processor": {
26
+ "do_convert_rgb": true,
27
+ "do_normalize": true,
28
+ "do_rescale": true,
29
+ "fps": 2.0,
30
+ "image_mean": [
31
+ 0.5,
32
+ 0.5,
33
+ 0.5
34
+ ],
35
+ "image_std": [
36
+ 0.5,
37
+ 0.5,
38
+ 0.5
39
+ ],
40
+ "max_frames": 768,
41
+ "max_pixels": 25165824,
42
+ "merge_size": 2,
43
+ "min_frames": 4,
44
+ "min_pixels": 4096,
45
+ "patch_size": 16,
46
+ "rescale_factor": 0.00392156862745098,
47
+ "temporal_patch_size": 2,
48
+ "video_processor_type": "Qwen3VLVideoProcessor"
49
+ }
50
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
3
+ size 19989325
tokenizer_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|im_end|>",
10
+ "errors": "replace",
11
+ "image_token": "<|image_pad|>",
12
+ "is_local": true,
13
+ "local_files_only": false,
14
+ "model_max_length": 262144,
15
+ "model_specific_special_tokens": {
16
+ "audio_bos_token": "<|audio_start|>",
17
+ "audio_eos_token": "<|audio_end|>",
18
+ "audio_token": "<|audio_pad|>",
19
+ "image_token": "<|image_pad|>",
20
+ "video_token": "<|video_pad|>",
21
+ "vision_bos_token": "<|vision_start|>",
22
+ "vision_eos_token": "<|vision_end|>"
23
+ },
24
+ "pad_token": "<|endoftext|>",
25
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
26
+ "processor_class": "Qwen3VLProcessor",
27
+ "split_special_tokens": false,
28
+ "tokenizer_class": "Qwen2Tokenizer",
29
+ "unk_token": null,
30
+ "video_token": "<|video_pad|>",
31
+ "vision_bos_token": "<|vision_start|>",
32
+ "vision_eos_token": "<|vision_end|>"
33
+ }
video_preprocessor_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "size": {
3
+ "longest_edge": 25165824,
4
+ "shortest_edge": 4096
5
+ },
6
+ "patch_size": 16,
7
+ "temporal_patch_size": 2,
8
+ "merge_size": 2,
9
+ "image_mean": [
10
+ 0.5,
11
+ 0.5,
12
+ 0.5
13
+ ],
14
+ "image_std": [
15
+ 0.5,
16
+ 0.5,
17
+ 0.5
18
+ ],
19
+ "processor_class": "Qwen3VLProcessor",
20
+ "video_processor_type": "Qwen3VLVideoProcessor"
21
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff