Hugging Face's logo

iAcloud
/
Qwen3-4B-INT8-RBLN

qwen3
compressed-tensors
Model card Files
xet
 Community
iAcloudEthan commited on
Commit
bc3a3f8
·
verified ·
1 Parent(s): 3a8cdc2

Upload 15 files

Browse files
Files changed (16) hide show
  1. .gitattributes +6 -0
  2. added_tokens.json +28 -0
  3. chat_template.jinja +89 -0
  4. config.json +113 -0
  5. decoder_batch_1.rbln +3 -0
  6. decoder_batch_12.rbln +3 -0
  7. decoder_batch_4.rbln +3 -0
  8. decoder_batch_8.rbln +3 -0
  9. generation_config.json +13 -0
  10. merges.txt +0 -0
  11. prefill.rbln +3 -0
  12. rbln_config.json +4736 -0
  13. special_tokens_map.json +31 -0
  14. tokenizer.json +3 -0
  15. tokenizer_config.json +243 -0
  16. vocab.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ decoder_batch_1.rbln filter=lfs diff=lfs merge=lfs -text
37
+ decoder_batch_12.rbln filter=lfs diff=lfs merge=lfs -text
38
+ decoder_batch_4.rbln filter=lfs diff=lfs merge=lfs -text
39
+ decoder_batch_8.rbln filter=lfs diff=lfs merge=lfs -text
40
+ prefill.rbln filter=lfs diff=lfs merge=lfs -text
41
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</think>": 151668,
3
+ "</tool_call>": 151658,
4
+ "</tool_response>": 151666,
5
+ "<think>": 151667,
6
+ "<tool_call>": 151657,
7
+ "<tool_response>": 151665,
8
+ "<|box_end|>": 151649,
9
+ "<|box_start|>": 151648,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|image_pad|>": 151655,
19
+ "<|object_ref_end|>": 151647,
20
+ "<|object_ref_start|>": 151646,
21
+ "<|quad_end|>": 151651,
22
+ "<|quad_start|>": 151650,
23
+ "<|repo_name|>": 151663,
24
+ "<|video_pad|>": 151656,
25
+ "<|vision_end|>": 151653,
26
+ "<|vision_pad|>": 151654,
27
+ "<|vision_start|>": 151652
28
+ }
chat_template.jinja ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0].role == 'system' %}
4
+ {{- messages[0].content + '\n\n' }}
5
+ {%- endif %}
6
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
7
+ {%- for tool in tools %}
8
+ {{- "\n" }}
9
+ {{- tool | tojson }}
10
+ {%- endfor %}
11
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
12
+ {%- else %}
13
+ {%- if messages[0].role == 'system' %}
14
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
15
+ {%- endif %}
16
+ {%- endif %}
17
+ {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
18
+ {%- for message in messages[::-1] %}
19
+ {%- set index = (messages|length - 1) - loop.index0 %}
20
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
21
+ {%- set ns.multi_step_tool = false %}
22
+ {%- set ns.last_query_index = index %}
23
+ {%- endif %}
24
+ {%- endfor %}
25
+ {%- for message in messages %}
26
+ {%- if message.content is string %}
27
+ {%- set content = message.content %}
28
+ {%- else %}
29
+ {%- set content = '' %}
30
+ {%- endif %}
31
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
32
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
33
+ {%- elif message.role == "assistant" %}
34
+ {%- set reasoning_content = '' %}
35
+ {%- if message.reasoning_content is string %}
36
+ {%- set reasoning_content = message.reasoning_content %}
37
+ {%- else %}
38
+ {%- if '</think>' in content %}
39
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
40
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
41
+ {%- endif %}
42
+ {%- endif %}
43
+ {%- if loop.index0 > ns.last_query_index %}
44
+ {%- if loop.last or (not loop.last and reasoning_content) %}
45
+ {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
46
+ {%- else %}
47
+ {{- '<|im_start|>' + message.role + '\n' + content }}
48
+ {%- endif %}
49
+ {%- else %}
50
+ {{- '<|im_start|>' + message.role + '\n' + content }}
51
+ {%- endif %}
52
+ {%- if message.tool_calls %}
53
+ {%- for tool_call in message.tool_calls %}
54
+ {%- if (loop.first and content) or (not loop.first) %}
55
+ {{- '\n' }}
56
+ {%- endif %}
57
+ {%- if tool_call.function %}
58
+ {%- set tool_call = tool_call.function %}
59
+ {%- endif %}
60
+ {{- '<tool_call>\n{"name": "' }}
61
+ {{- tool_call.name }}
62
+ {{- '", "arguments": ' }}
63
+ {%- if tool_call.arguments is string %}
64
+ {{- tool_call.arguments }}
65
+ {%- else %}
66
+ {{- tool_call.arguments | tojson }}
67
+ {%- endif %}
68
+ {{- '}\n</tool_call>' }}
69
+ {%- endfor %}
70
+ {%- endif %}
71
+ {{- '<|im_end|>\n' }}
72
+ {%- elif message.role == "tool" %}
73
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
74
+ {{- '<|im_start|>user' }}
75
+ {%- endif %}
76
+ {{- '\n<tool_response>\n' }}
77
+ {{- content }}
78
+ {{- '\n</tool_response>' }}
79
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
80
+ {{- '<|im_end|>\n' }}
81
+ {%- endif %}
82
+ {%- endif %}
83
+ {%- endfor %}
84
+ {%- if add_generation_prompt %}
85
+ {{- '<|im_start|>assistant\n' }}
86
+ {%- if enable_thinking is defined and enable_thinking is false %}
87
+ {{- '<think>\n\n</think>\n\n' }}
88
+ {%- endif %}
89
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 151645,
10
+ "head_dim": 128,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 2560,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 9728,
15
+ "layer_types": [
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention",
42
+ "full_attention",
43
+ "full_attention",
44
+ "full_attention",
45
+ "full_attention",
46
+ "full_attention",
47
+ "full_attention",
48
+ "full_attention",
49
+ "full_attention",
50
+ "full_attention",
51
+ "full_attention"
52
+ ],
53
+ "max_position_embeddings": 40960,
54
+ "max_window_layers": 36,
55
+ "model_type": "qwen3",
56
+ "num_attention_heads": 32,
57
+ "num_hidden_layers": 36,
58
+ "num_key_value_heads": 8,
59
+ "quantization_config": {
60
+ "config_groups": {
61
+ "group_0": {
62
+ "format": "int-quantized",
63
+ "input_activations": {
64
+ "actorder": null,
65
+ "block_structure": null,
66
+ "dynamic": true,
67
+ "group_size": null,
68
+ "num_bits": 8,
69
+ "observer": null,
70
+ "observer_kwargs": {},
71
+ "strategy": "token",
72
+ "symmetric": true,
73
+ "type": "int"
74
+ },
75
+ "output_activations": null,
76
+ "targets": [
77
+ "Linear"
78
+ ],
79
+ "weights": {
80
+ "actorder": null,
81
+ "block_structure": null,
82
+ "dynamic": false,
83
+ "group_size": null,
84
+ "num_bits": 8,
85
+ "observer": "minmax",
86
+ "observer_kwargs": {},
87
+ "strategy": "channel",
88
+ "symmetric": true,
89
+ "type": "int"
90
+ }
91
+ }
92
+ },
93
+ "format": "int-quantized",
94
+ "global_compression_ratio": null,
95
+ "ignore": [
96
+ "lm_head"
97
+ ],
98
+ "kv_cache_scheme": null,
99
+ "quant_method": "compressed-tensors",
100
+ "quantization_status": "compressed",
101
+ "sparsity_config": {},
102
+ "transform_config": {}
103
+ },
104
+ "rms_norm_eps": 1e-06,
105
+ "rope_scaling": null,
106
+ "rope_theta": 1000000,
107
+ "sliding_window": null,
108
+ "tie_word_embeddings": true,
109
+ "transformers_version": "4.57.3",
110
+ "use_cache": true,
111
+ "use_sliding_window": false,
112
+ "vocab_size": 151936
113
+ }
decoder_batch_1.rbln ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c7f6a08009f2634de6119d7dd8b16b69138f15248dbcfd5313f32b7687d2330
3
+ size 816545423
decoder_batch_12.rbln ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a16d30cce0e0039c963588b19ac457092d99dc9c96d8bdae8760c3b1b427826a
3
+ size 957668918
decoder_batch_4.rbln ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:980e8d2058c0c052590605d0efc06f69aa1198177a51938941939655c74c4480
3
+ size 859333101
decoder_batch_8.rbln ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4c3a88764cd1b109800fb6151cd0c037f0c11123ddf39012568a81a0895c817
3
+ size 912130568
generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 151645,
6
+ 151643
7
+ ],
8
+ "pad_token_id": 151643,
9
+ "temperature": 0.6,
10
+ "top_k": 20,
11
+ "top_p": 0.95,
12
+ "transformers_version": "4.57.3"
13
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
prefill.rbln ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0d0ec6952527315bfac468a30b6e36d0e2fb7dbc7e67ff5a657e8fa1c5a2da5
3
+ size 5397384449
rbln_config.json ADDED
@@ -0,0 +1,4736 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_name": "RBLNQwen3ForCausalLMConfig",
3
+ "dtype": "float16",
4
+ "optimum_rbln_version": "0.10.0.post2",
5
+ "_compile_cfgs": [
6
+ {
7
+ "compiled_model_name": "prefill",
8
+ "input_info": [
9
+ [
10
+ "input_ids",
11
+ [
12
+ 1,
13
+ 128
14
+ ],
15
+ "int64"
16
+ ],
17
+ [
18
+ "cache_position",
19
+ [
20
+ 1,
21
+ 128
22
+ ],
23
+ "int32"
24
+ ],
25
+ [
26
+ "block_tables",
27
+ [
28
+ 2
29
+ ],
30
+ "int16"
31
+ ],
32
+ [
33
+ "query_position",
34
+ [],
35
+ "int16"
36
+ ],
37
+ [
38
+ "past_key_values_0",
39
+ [
40
+ 1,
41
+ 8,
42
+ 16384,
43
+ 128
44
+ ],
45
+ "float16"
46
+ ],
47
+ [
48
+ "past_key_values_1",
49
+ [
50
+ 1,
51
+ 8,
52
+ 16384,
53
+ 128
54
+ ],
55
+ "float16"
56
+ ],
57
+ [
58
+ "past_key_values_2",
59
+ [
60
+ 1,
61
+ 8,
62
+ 16384,
63
+ 128
64
+ ],
65
+ "float16"
66
+ ],
67
+ [
68
+ "past_key_values_3",
69
+ [
70
+ 1,
71
+ 8,
72
+ 16384,
73
+ 128
74
+ ],
75
+ "float16"
76
+ ],
77
+ [
78
+ "past_key_values_4",
79
+ [
80
+ 1,
81
+ 8,
82
+ 16384,
83
+ 128
84
+ ],
85
+ "float16"
86
+ ],
87
+ [
88
+ "past_key_values_5",
89
+ [
90
+ 1,
91
+ 8,
92
+ 16384,
93
+ 128
94
+ ],
95
+ "float16"
96
+ ],
97
+ [
98
+ "past_key_values_6",
99
+ [
100
+ 1,
101
+ 8,
102
+ 16384,
103
+ 128
104
+ ],
105
+ "float16"
106
+ ],
107
+ [
108
+ "past_key_values_7",
109
+ [
110
+ 1,
111
+ 8,
112
+ 16384,
113
+ 128
114
+ ],
115
+ "float16"
116
+ ],
117
+ [
118
+ "past_key_values_8",
119
+ [
120
+ 1,
121
+ 8,
122
+ 16384,
123
+ 128
124
+ ],
125
+ "float16"
126
+ ],
127
+ [
128
+ "past_key_values_9",
129
+ [
130
+ 1,
131
+ 8,
132
+ 16384,
133
+ 128
134
+ ],
135
+ "float16"
136
+ ],
137
+ [
138
+ "past_key_values_10",
139
+ [
140
+ 1,
141
+ 8,
142
+ 16384,
143
+ 128
144
+ ],
145
+ "float16"
146
+ ],
147
+ [
148
+ "past_key_values_11",
149
+ [
150
+ 1,
151
+ 8,
152
+ 16384,
153
+ 128
154
+ ],
155
+ "float16"
156
+ ],
157
+ [
158
+ "past_key_values_12",
159
+ [
160
+ 1,
161
+ 8,
162
+ 16384,
163
+ 128
164
+ ],
165
+ "float16"
166
+ ],
167
+ [
168
+ "past_key_values_13",
169
+ [
170
+ 1,
171
+ 8,
172
+ 16384,
173
+ 128
174
+ ],
175
+ "float16"
176
+ ],
177
+ [
178
+ "past_key_values_14",
179
+ [
180
+ 1,
181
+ 8,
182
+ 16384,
183
+ 128
184
+ ],
185
+ "float16"
186
+ ],
187
+ [
188
+ "past_key_values_15",
189
+ [
190
+ 1,
191
+ 8,
192
+ 16384,
193
+ 128
194
+ ],
195
+ "float16"
196
+ ],
197
+ [
198
+ "past_key_values_16",
199
+ [
200
+ 1,
201
+ 8,
202
+ 16384,
203
+ 128
204
+ ],
205
+ "float16"
206
+ ],
207
+ [
208
+ "past_key_values_17",
209
+ [
210
+ 1,
211
+ 8,
212
+ 16384,
213
+ 128
214
+ ],
215
+ "float16"
216
+ ],
217
+ [
218
+ "past_key_values_18",
219
+ [
220
+ 1,
221
+ 8,
222
+ 16384,
223
+ 128
224
+ ],
225
+ "float16"
226
+ ],
227
+ [
228
+ "past_key_values_19",
229
+ [
230
+ 1,
231
+ 8,
232
+ 16384,
233
+ 128
234
+ ],
235
+ "float16"
236
+ ],
237
+ [
238
+ "past_key_values_20",
239
+ [
240
+ 1,
241
+ 8,
242
+ 16384,
243
+ 128
244
+ ],
245
+ "float16"
246
+ ],
247
+ [
248
+ "past_key_values_21",
249
+ [
250
+ 1,
251
+ 8,
252
+ 16384,
253
+ 128
254
+ ],
255
+ "float16"
256
+ ],
257
+ [
258
+ "past_key_values_22",
259
+ [
260
+ 1,
261
+ 8,
262
+ 16384,
263
+ 128
264
+ ],
265
+ "float16"
266
+ ],
267
+ [
268
+ "past_key_values_23",
269
+ [
270
+ 1,
271
+ 8,
272
+ 16384,
273
+ 128
274
+ ],
275
+ "float16"
276
+ ],
277
+ [
278
+ "past_key_values_24",
279
+ [
280
+ 1,
281
+ 8,
282
+ 16384,
283
+ 128
284
+ ],
285
+ "float16"
286
+ ],
287
+ [
288
+ "past_key_values_25",
289
+ [
290
+ 1,
291
+ 8,
292
+ 16384,
293
+ 128
294
+ ],
295
+ "float16"
296
+ ],
297
+ [
298
+ "past_key_values_26",
299
+ [
300
+ 1,
301
+ 8,
302
+ 16384,
303
+ 128
304
+ ],
305
+ "float16"
306
+ ],
307
+ [
308
+ "past_key_values_27",
309
+ [
310
+ 1,
311
+ 8,
312
+ 16384,
313
+ 128
314
+ ],
315
+ "float16"
316
+ ],
317
+ [
318
+ "past_key_values_28",
319
+ [
320
+ 1,
321
+ 8,
322
+ 16384,
323
+ 128
324
+ ],
325
+ "float16"
326
+ ],
327
+ [
328
+ "past_key_values_29",
329
+ [
330
+ 1,
331
+ 8,
332
+ 16384,
333
+ 128
334
+ ],
335
+ "float16"
336
+ ],
337
+ [
338
+ "past_key_values_30",
339
+ [
340
+ 1,
341
+ 8,
342
+ 16384,
343
+ 128
344
+ ],
345
+ "float16"
346
+ ],
347
+ [
348
+ "past_key_values_31",
349
+ [
350
+ 1,
351
+ 8,
352
+ 16384,
353
+ 128
354
+ ],
355
+ "float16"
356
+ ],
357
+ [
358
+ "past_key_values_32",
359
+ [
360
+ 1,
361
+ 8,
362
+ 16384,
363
+ 128
364
+ ],
365
+ "float16"
366
+ ],
367
+ [
368
+ "past_key_values_33",
369
+ [
370
+ 1,
371
+ 8,
372
+ 16384,
373
+ 128
374
+ ],
375
+ "float16"
376
+ ],
377
+ [
378
+ "past_key_values_34",
379
+ [
380
+ 1,
381
+ 8,
382
+ 16384,
383
+ 128
384
+ ],
385
+ "float16"
386
+ ],
387
+ [
388
+ "past_key_values_35",
389
+ [
390
+ 1,
391
+ 8,
392
+ 16384,
393
+ 128
394
+ ],
395
+ "float16"
396
+ ],
397
+ [
398
+ "past_key_values_36",
399
+ [
400
+ 1,
401
+ 8,
402
+ 16384,
403
+ 128
404
+ ],
405
+ "float16"
406
+ ],
407
+ [
408
+ "past_key_values_37",
409
+ [
410
+ 1,
411
+ 8,
412
+ 16384,
413
+ 128
414
+ ],
415
+ "float16"
416
+ ],
417
+ [
418
+ "past_key_values_38",
419
+ [
420
+ 1,
421
+ 8,
422
+ 16384,
423
+ 128
424
+ ],
425
+ "float16"
426
+ ],
427
+ [
428
+ "past_key_values_39",
429
+ [
430
+ 1,
431
+ 8,
432
+ 16384,
433
+ 128
434
+ ],
435
+ "float16"
436
+ ],
437
+ [
438
+ "past_key_values_40",
439
+ [
440
+ 1,
441
+ 8,
442
+ 16384,
443
+ 128
444
+ ],
445
+ "float16"
446
+ ],
447
+ [
448
+ "past_key_values_41",
449
+ [
450
+ 1,
451
+ 8,
452
+ 16384,
453
+ 128
454
+ ],
455
+ "float16"
456
+ ],
457
+ [
458
+ "past_key_values_42",
459
+ [
460
+ 1,
461
+ 8,
462
+ 16384,
463
+ 128
464
+ ],
465
+ "float16"
466
+ ],
467
+ [
468
+ "past_key_values_43",
469
+ [
470
+ 1,
471
+ 8,
472
+ 16384,
473
+ 128
474
+ ],
475
+ "float16"
476
+ ],
477
+ [
478
+ "past_key_values_44",
479
+ [
480
+ 1,
481
+ 8,
482
+ 16384,
483
+ 128
484
+ ],
485
+ "float16"
486
+ ],
487
+ [
488
+ "past_key_values_45",
489
+ [
490
+ 1,
491
+ 8,
492
+ 16384,
493
+ 128
494
+ ],
495
+ "float16"
496
+ ],
497
+ [
498
+ "past_key_values_46",
499
+ [
500
+ 1,
501
+ 8,
502
+ 16384,
503
+ 128
504
+ ],
505
+ "float16"
506
+ ],
507
+ [
508
+ "past_key_values_47",
509
+ [
510
+ 1,
511
+ 8,
512
+ 16384,
513
+ 128
514
+ ],
515
+ "float16"
516
+ ],
517
+ [
518
+ "past_key_values_48",
519
+ [
520
+ 1,
521
+ 8,
522
+ 16384,
523
+ 128
524
+ ],
525
+ "float16"
526
+ ],
527
+ [
528
+ "past_key_values_49",
529
+ [
530
+ 1,
531
+ 8,
532
+ 16384,
533
+ 128
534
+ ],
535
+ "float16"
536
+ ],
537
+ [
538
+ "past_key_values_50",
539
+ [
540
+ 1,
541
+ 8,
542
+ 16384,
543
+ 128
544
+ ],
545
+ "float16"
546
+ ],
547
+ [
548
+ "past_key_values_51",
549
+ [
550
+ 1,
551
+ 8,
552
+ 16384,
553
+ 128
554
+ ],
555
+ "float16"
556
+ ],
557
+ [
558
+ "past_key_values_52",
559
+ [
560
+ 1,
561
+ 8,
562
+ 16384,
563
+ 128
564
+ ],
565
+ "float16"
566
+ ],
567
+ [
568
+ "past_key_values_53",
569
+ [
570
+ 1,
571
+ 8,
572
+ 16384,
573
+ 128
574
+ ],
575
+ "float16"
576
+ ],
577
+ [
578
+ "past_key_values_54",
579
+ [
580
+ 1,
581
+ 8,
582
+ 16384,
583
+ 128
584
+ ],
585
+ "float16"
586
+ ],
587
+ [
588
+ "past_key_values_55",
589
+ [
590
+ 1,
591
+ 8,
592
+ 16384,
593
+ 128
594
+ ],
595
+ "float16"
596
+ ],
597
+ [
598
+ "past_key_values_56",
599
+ [
600
+ 1,
601
+ 8,
602
+ 16384,
603
+ 128
604
+ ],
605
+ "float16"
606
+ ],
607
+ [
608
+ "past_key_values_57",
609
+ [
610
+ 1,
611
+ 8,
612
+ 16384,
613
+ 128
614
+ ],
615
+ "float16"
616
+ ],
617
+ [
618
+ "past_key_values_58",
619
+ [
620
+ 1,
621
+ 8,
622
+ 16384,
623
+ 128
624
+ ],
625
+ "float16"
626
+ ],
627
+ [
628
+ "past_key_values_59",
629
+ [
630
+ 1,
631
+ 8,
632
+ 16384,
633
+ 128
634
+ ],
635
+ "float16"
636
+ ],
637
+ [
638
+ "past_key_values_60",
639
+ [
640
+ 1,
641
+ 8,
642
+ 16384,
643
+ 128
644
+ ],
645
+ "float16"
646
+ ],
647
+ [
648
+ "past_key_values_61",
649
+ [
650
+ 1,
651
+ 8,
652
+ 16384,
653
+ 128
654
+ ],
655
+ "float16"
656
+ ],
657
+ [
658
+ "past_key_values_62",
659
+ [
660
+ 1,
661
+ 8,
662
+ 16384,
663
+ 128
664
+ ],
665
+ "float16"
666
+ ],
667
+ [
668
+ "past_key_values_63",
669
+ [
670
+ 1,
671
+ 8,
672
+ 16384,
673
+ 128
674
+ ],
675
+ "float16"
676
+ ],
677
+ [
678
+ "past_key_values_64",
679
+ [
680
+ 1,
681
+ 8,
682
+ 16384,
683
+ 128
684
+ ],
685
+ "float16"
686
+ ],
687
+ [
688
+ "past_key_values_65",
689
+ [
690
+ 1,
691
+ 8,
692
+ 16384,
693
+ 128
694
+ ],
695
+ "float16"
696
+ ],
697
+ [
698
+ "past_key_values_66",
699
+ [
700
+ 1,
701
+ 8,
702
+ 16384,
703
+ 128
704
+ ],
705
+ "float16"
706
+ ],
707
+ [
708
+ "past_key_values_67",
709
+ [
710
+ 1,
711
+ 8,
712
+ 16384,
713
+ 128
714
+ ],
715
+ "float16"
716
+ ],
717
+ [
718
+ "past_key_values_68",
719
+ [
720
+ 1,
721
+ 8,
722
+ 16384,
723
+ 128
724
+ ],
725
+ "float16"
726
+ ],
727
+ [
728
+ "past_key_values_69",
729
+ [
730
+ 1,
731
+ 8,
732
+ 16384,
733
+ 128
734
+ ],
735
+ "float16"
736
+ ],
737
+ [
738
+ "past_key_values_70",
739
+ [
740
+ 1,
741
+ 8,
742
+ 16384,
743
+ 128
744
+ ],
745
+ "float16"
746
+ ],
747
+ [
748
+ "past_key_values_71",
749
+ [
750
+ 1,
751
+ 8,
752
+ 16384,
753
+ 128
754
+ ],
755
+ "float16"
756
+ ]
757
+ ],
758
+ "npu": "RBLN-CA25",
759
+ "tensor_parallel_size": 4
760
+ },
761
+ {
762
+ "compiled_model_name": "decoder_batch_12",
763
+ "input_info": [
764
+ [
765
+ "input_ids",
766
+ [
767
+ 12,
768
+ 1
769
+ ],
770
+ "int64"
771
+ ],
772
+ [
773
+ "cache_position",
774
+ [
775
+ 12,
776
+ 1
777
+ ],
778
+ "int32"
779
+ ],
780
+ [
781
+ "block_tables",
782
+ [
783
+ 12,
784
+ 2
785
+ ],
786
+ "int16"
787
+ ],
788
+ [
789
+ "past_key_values_0",
790
+ [
791
+ 1,
792
+ 8,
793
+ 16384,
794
+ 128
795
+ ],
796
+ "float16"
797
+ ],
798
+ [
799
+ "past_key_values_1",
800
+ [
801
+ 1,
802
+ 8,
803
+ 16384,
804
+ 128
805
+ ],
806
+ "float16"
807
+ ],
808
+ [
809
+ "past_key_values_2",
810
+ [
811
+ 1,
812
+ 8,
813
+ 16384,
814
+ 128
815
+ ],
816
+ "float16"
817
+ ],
818
+ [
819
+ "past_key_values_3",
820
+ [
821
+ 1,
822
+ 8,
823
+ 16384,
824
+ 128
825
+ ],
826
+ "float16"
827
+ ],
828
+ [
829
+ "past_key_values_4",
830
+ [
831
+ 1,
832
+ 8,
833
+ 16384,
834
+ 128
835
+ ],
836
+ "float16"
837
+ ],
838
+ [
839
+ "past_key_values_5",
840
+ [
841
+ 1,
842
+ 8,
843
+ 16384,
844
+ 128
845
+ ],
846
+ "float16"
847
+ ],
848
+ [
849
+ "past_key_values_6",
850
+ [
851
+ 1,
852
+ 8,
853
+ 16384,
854
+ 128
855
+ ],
856
+ "float16"
857
+ ],
858
+ [
859
+ "past_key_values_7",
860
+ [
861
+ 1,
862
+ 8,
863
+ 16384,
864
+ 128
865
+ ],
866
+ "float16"
867
+ ],
868
+ [
869
+ "past_key_values_8",
870
+ [
871
+ 1,
872
+ 8,
873
+ 16384,
874
+ 128
875
+ ],
876
+ "float16"
877
+ ],
878
+ [
879
+ "past_key_values_9",
880
+ [
881
+ 1,
882
+ 8,
883
+ 16384,
884
+ 128
885
+ ],
886
+ "float16"
887
+ ],
888
+ [
889
+ "past_key_values_10",
890
+ [
891
+ 1,
892
+ 8,
893
+ 16384,
894
+ 128
895
+ ],
896
+ "float16"
897
+ ],
898
+ [
899
+ "past_key_values_11",
900
+ [
901
+ 1,
902
+ 8,
903
+ 16384,
904
+ 128
905
+ ],
906
+ "float16"
907
+ ],
908
+ [
909
+ "past_key_values_12",
910
+ [
911
+ 1,
912
+ 8,
913
+ 16384,
914
+ 128
915
+ ],
916
+ "float16"
917
+ ],
918
+ [
919
+ "past_key_values_13",
920
+ [
921
+ 1,
922
+ 8,
923
+ 16384,
924
+ 128
925
+ ],
926
+ "float16"
927
+ ],
928
+ [
929
+ "past_key_values_14",
930
+ [
931
+ 1,
932
+ 8,
933
+ 16384,
934
+ 128
935
+ ],
936
+ "float16"
937
+ ],
938
+ [
939
+ "past_key_values_15",
940
+ [
941
+ 1,
942
+ 8,
943
+ 16384,
944
+ 128
945
+ ],
946
+ "float16"
947
+ ],
948
+ [
949
+ "past_key_values_16",
950
+ [
951
+ 1,
952
+ 8,
953
+ 16384,
954
+ 128
955
+ ],
956
+ "float16"
957
+ ],
958
+ [
959
+ "past_key_values_17",
960
+ [
961
+ 1,
962
+ 8,
963
+ 16384,
964
+ 128
965
+ ],
966
+ "float16"
967
+ ],
968
+ [
969
+ "past_key_values_18",
970
+ [
971
+ 1,
972
+ 8,
973
+ 16384,
974
+ 128
975
+ ],
976
+ "float16"
977
+ ],
978
+ [
979
+ "past_key_values_19",
980
+ [
981
+ 1,
982
+ 8,
983
+ 16384,
984
+ 128
985
+ ],
986
+ "float16"
987
+ ],
988
+ [
989
+ "past_key_values_20",
990
+ [
991
+ 1,
992
+ 8,
993
+ 16384,
994
+ 128
995
+ ],
996
+ "float16"
997
+ ],
998
+ [
999
+ "past_key_values_21",
1000
+ [
1001
+ 1,
1002
+ 8,
1003
+ 16384,
1004
+ 128
1005
+ ],
1006
+ "float16"
1007
+ ],
1008
+ [
1009
+ "past_key_values_22",
1010
+ [
1011
+ 1,
1012
+ 8,
1013
+ 16384,
1014
+ 128
1015
+ ],
1016
+ "float16"
1017
+ ],
1018
+ [
1019
+ "past_key_values_23",
1020
+ [
1021
+ 1,
1022
+ 8,
1023
+ 16384,
1024
+ 128
1025
+ ],
1026
+ "float16"
1027
+ ],
1028
+ [
1029
+ "past_key_values_24",
1030
+ [
1031
+ 1,
1032
+ 8,
1033
+ 16384,
1034
+ 128
1035
+ ],
1036
+ "float16"
1037
+ ],
1038
+ [
1039
+ "past_key_values_25",
1040
+ [
1041
+ 1,
1042
+ 8,
1043
+ 16384,
1044
+ 128
1045
+ ],
1046
+ "float16"
1047
+ ],
1048
+ [
1049
+ "past_key_values_26",
1050
+ [
1051
+ 1,
1052
+ 8,
1053
+ 16384,
1054
+ 128
1055
+ ],
1056
+ "float16"
1057
+ ],
1058
+ [
1059
+ "past_key_values_27",
1060
+ [
1061
+ 1,
1062
+ 8,
1063
+ 16384,
1064
+ 128
1065
+ ],
1066
+ "float16"
1067
+ ],
1068
+ [
1069
+ "past_key_values_28",
1070
+ [
1071
+ 1,
1072
+ 8,
1073
+ 16384,
1074
+ 128
1075
+ ],
1076
+ "float16"
1077
+ ],
1078
+ [
1079
+ "past_key_values_29",
1080
+ [
1081
+ 1,
1082
+ 8,
1083
+ 16384,
1084
+ 128
1085
+ ],
1086
+ "float16"
1087
+ ],
1088
+ [
1089
+ "past_key_values_30",
1090
+ [
1091
+ 1,
1092
+ 8,
1093
+ 16384,
1094
+ 128
1095
+ ],
1096
+ "float16"
1097
+ ],
1098
+ [
1099
+ "past_key_values_31",
1100
+ [
1101
+ 1,
1102
+ 8,
1103
+ 16384,
1104
+ 128
1105
+ ],
1106
+ "float16"
1107
+ ],
1108
+ [
1109
+ "past_key_values_32",
1110
+ [
1111
+ 1,
1112
+ 8,
1113
+ 16384,
1114
+ 128
1115
+ ],
1116
+ "float16"
1117
+ ],
1118
+ [
1119
+ "past_key_values_33",
1120
+ [
1121
+ 1,
1122
+ 8,
1123
+ 16384,
1124
+ 128
1125
+ ],
1126
+ "float16"
1127
+ ],
1128
+ [
1129
+ "past_key_values_34",
1130
+ [
1131
+ 1,
1132
+ 8,
1133
+ 16384,
1134
+ 128
1135
+ ],
1136
+ "float16"
1137
+ ],
1138
+ [
1139
+ "past_key_values_35",
1140
+ [
1141
+ 1,
1142
+ 8,
1143
+ 16384,
1144
+ 128
1145
+ ],
1146
+ "float16"
1147
+ ],
1148
+ [
1149
+ "past_key_values_36",
1150
+ [
1151
+ 1,
1152
+ 8,
1153
+ 16384,
1154
+ 128
1155
+ ],
1156
+ "float16"
1157
+ ],
1158
+ [
1159
+ "past_key_values_37",
1160
+ [
1161
+ 1,
1162
+ 8,
1163
+ 16384,
1164
+ 128
1165
+ ],
1166
+ "float16"
1167
+ ],
1168
+ [
1169
+ "past_key_values_38",
1170
+ [
1171
+ 1,
1172
+ 8,
1173
+ 16384,
1174
+ 128
1175
+ ],
1176
+ "float16"
1177
+ ],
1178
+ [
1179
+ "past_key_values_39",
1180
+ [
1181
+ 1,
1182
+ 8,
1183
+ 16384,
1184
+ 128
1185
+ ],
1186
+ "float16"
1187
+ ],
1188
+ [
1189
+ "past_key_values_40",
1190
+ [
1191
+ 1,
1192
+ 8,
1193
+ 16384,
1194
+ 128
1195
+ ],
1196
+ "float16"
1197
+ ],
1198
+ [
1199
+ "past_key_values_41",
1200
+ [
1201
+ 1,
1202
+ 8,
1203
+ 16384,
1204
+ 128
1205
+ ],
1206
+ "float16"
1207
+ ],
1208
+ [
1209
+ "past_key_values_42",
1210
+ [
1211
+ 1,
1212
+ 8,
1213
+ 16384,
1214
+ 128
1215
+ ],
1216
+ "float16"
1217
+ ],
1218
+ [
1219
+ "past_key_values_43",
1220
+ [
1221
+ 1,
1222
+ 8,
1223
+ 16384,
1224
+ 128
1225
+ ],
1226
+ "float16"
1227
+ ],
1228
+ [
1229
+ "past_key_values_44",
1230
+ [
1231
+ 1,
1232
+ 8,
1233
+ 16384,
1234
+ 128
1235
+ ],
1236
+ "float16"
1237
+ ],
1238
+ [
1239
+ "past_key_values_45",
1240
+ [
1241
+ 1,
1242
+ 8,
1243
+ 16384,
1244
+ 128
1245
+ ],
1246
+ "float16"
1247
+ ],
1248
+ [
1249
+ "past_key_values_46",
1250
+ [
1251
+ 1,
1252
+ 8,
1253
+ 16384,
1254
+ 128
1255
+ ],
1256
+ "float16"
1257
+ ],
1258
+ [
1259
+ "past_key_values_47",
1260
+ [
1261
+ 1,
1262
+ 8,
1263
+ 16384,
1264
+ 128
1265
+ ],
1266
+ "float16"
1267
+ ],
1268
+ [
1269
+ "past_key_values_48",
1270
+ [
1271
+ 1,
1272
+ 8,
1273
+ 16384,
1274
+ 128
1275
+ ],
1276
+ "float16"
1277
+ ],
1278
+ [
1279
+ "past_key_values_49",
1280
+ [
1281
+ 1,
1282
+ 8,
1283
+ 16384,
1284
+ 128
1285
+ ],
1286
+ "float16"
1287
+ ],
1288
+ [
1289
+ "past_key_values_50",
1290
+ [
1291
+ 1,
1292
+ 8,
1293
+ 16384,
1294
+ 128
1295
+ ],
1296
+ "float16"
1297
+ ],
1298
+ [
1299
+ "past_key_values_51",
1300
+ [
1301
+ 1,
1302
+ 8,
1303
+ 16384,
1304
+ 128
1305
+ ],
1306
+ "float16"
1307
+ ],
1308
+ [
1309
+ "past_key_values_52",
1310
+ [
1311
+ 1,
1312
+ 8,
1313
+ 16384,
1314
+ 128
1315
+ ],
1316
+ "float16"
1317
+ ],
1318
+ [
1319
+ "past_key_values_53",
1320
+ [
1321
+ 1,
1322
+ 8,
1323
+ 16384,
1324
+ 128
1325
+ ],
1326
+ "float16"
1327
+ ],
1328
+ [
1329
+ "past_key_values_54",
1330
+ [
1331
+ 1,
1332
+ 8,
1333
+ 16384,
1334
+ 128
1335
+ ],
1336
+ "float16"
1337
+ ],
1338
+ [
1339
+ "past_key_values_55",
1340
+ [
1341
+ 1,
1342
+ 8,
1343
+ 16384,
1344
+ 128
1345
+ ],
1346
+ "float16"
1347
+ ],
1348
+ [
1349
+ "past_key_values_56",
1350
+ [
1351
+ 1,
1352
+ 8,
1353
+ 16384,
1354
+ 128
1355
+ ],
1356
+ "float16"
1357
+ ],
1358
+ [
1359
+ "past_key_values_57",
1360
+ [
1361
+ 1,
1362
+ 8,
1363
+ 16384,
1364
+ 128
1365
+ ],
1366
+ "float16"
1367
+ ],
1368
+ [
1369
+ "past_key_values_58",
1370
+ [
1371
+ 1,
1372
+ 8,
1373
+ 16384,
1374
+ 128
1375
+ ],
1376
+ "float16"
1377
+ ],
1378
+ [
1379
+ "past_key_values_59",
1380
+ [
1381
+ 1,
1382
+ 8,
1383
+ 16384,
1384
+ 128
1385
+ ],
1386
+ "float16"
1387
+ ],
1388
+ [
1389
+ "past_key_values_60",
1390
+ [
1391
+ 1,
1392
+ 8,
1393
+ 16384,
1394
+ 128
1395
+ ],
1396
+ "float16"
1397
+ ],
1398
+ [
1399
+ "past_key_values_61",
1400
+ [
1401
+ 1,
1402
+ 8,
1403
+ 16384,
1404
+ 128
1405
+ ],
1406
+ "float16"
1407
+ ],
1408
+ [
1409
+ "past_key_values_62",
1410
+ [
1411
+ 1,
1412
+ 8,
1413
+ 16384,
1414
+ 128
1415
+ ],
1416
+ "float16"
1417
+ ],
1418
+ [
1419
+ "past_key_values_63",
1420
+ [
1421
+ 1,
1422
+ 8,
1423
+ 16384,
1424
+ 128
1425
+ ],
1426
+ "float16"
1427
+ ],
1428
+ [
1429
+ "past_key_values_64",
1430
+ [
1431
+ 1,
1432
+ 8,
1433
+ 16384,
1434
+ 128
1435
+ ],
1436
+ "float16"
1437
+ ],
1438
+ [
1439
+ "past_key_values_65",
1440
+ [
1441
+ 1,
1442
+ 8,
1443
+ 16384,
1444
+ 128
1445
+ ],
1446
+ "float16"
1447
+ ],
1448
+ [
1449
+ "past_key_values_66",
1450
+ [
1451
+ 1,
1452
+ 8,
1453
+ 16384,
1454
+ 128
1455
+ ],
1456
+ "float16"
1457
+ ],
1458
+ [
1459
+ "past_key_values_67",
1460
+ [
1461
+ 1,
1462
+ 8,
1463
+ 16384,
1464
+ 128
1465
+ ],
1466
+ "float16"
1467
+ ],
1468
+ [
1469
+ "past_key_values_68",
1470
+ [
1471
+ 1,
1472
+ 8,
1473
+ 16384,
1474
+ 128
1475
+ ],
1476
+ "float16"
1477
+ ],
1478
+ [
1479
+ "past_key_values_69",
1480
+ [
1481
+ 1,
1482
+ 8,
1483
+ 16384,
1484
+ 128
1485
+ ],
1486
+ "float16"
1487
+ ],
1488
+ [
1489
+ "past_key_values_70",
1490
+ [
1491
+ 1,
1492
+ 8,
1493
+ 16384,
1494
+ 128
1495
+ ],
1496
+ "float16"
1497
+ ],
1498
+ [
1499
+ "past_key_values_71",
1500
+ [
1501
+ 1,
1502
+ 8,
1503
+ 16384,
1504
+ 128
1505
+ ],
1506
+ "float16"
1507
+ ]
1508
+ ],
1509
+ "npu": "RBLN-CA25",
1510
+ "tensor_parallel_size": 4
1511
+ },
1512
+ {
1513
+ "compiled_model_name": "decoder_batch_8",
1514
+ "input_info": [
1515
+ [
1516
+ "input_ids",
1517
+ [
1518
+ 8,
1519
+ 1
1520
+ ],
1521
+ "int64"
1522
+ ],
1523
+ [
1524
+ "cache_position",
1525
+ [
1526
+ 8,
1527
+ 1
1528
+ ],
1529
+ "int32"
1530
+ ],
1531
+ [
1532
+ "block_tables",
1533
+ [
1534
+ 8,
1535
+ 2
1536
+ ],
1537
+ "int16"
1538
+ ],
1539
+ [
1540
+ "past_key_values_0",
1541
+ [
1542
+ 1,
1543
+ 8,
1544
+ 16384,
1545
+ 128
1546
+ ],
1547
+ "float16"
1548
+ ],
1549
+ [
1550
+ "past_key_values_1",
1551
+ [
1552
+ 1,
1553
+ 8,
1554
+ 16384,
1555
+ 128
1556
+ ],
1557
+ "float16"
1558
+ ],
1559
+ [
1560
+ "past_key_values_2",
1561
+ [
1562
+ 1,
1563
+ 8,
1564
+ 16384,
1565
+ 128
1566
+ ],
1567
+ "float16"
1568
+ ],
1569
+ [
1570
+ "past_key_values_3",
1571
+ [
1572
+ 1,
1573
+ 8,
1574
+ 16384,
1575
+ 128
1576
+ ],
1577
+ "float16"
1578
+ ],
1579
+ [
1580
+ "past_key_values_4",
1581
+ [
1582
+ 1,
1583
+ 8,
1584
+ 16384,
1585
+ 128
1586
+ ],
1587
+ "float16"
1588
+ ],
1589
+ [
1590
+ "past_key_values_5",
1591
+ [
1592
+ 1,
1593
+ 8,
1594
+ 16384,
1595
+ 128
1596
+ ],
1597
+ "float16"
1598
+ ],
1599
+ [
1600
+ "past_key_values_6",
1601
+ [
1602
+ 1,
1603
+ 8,
1604
+ 16384,
1605
+ 128
1606
+ ],
1607
+ "float16"
1608
+ ],
1609
+ [
1610
+ "past_key_values_7",
1611
+ [
1612
+ 1,
1613
+ 8,
1614
+ 16384,
1615
+ 128
1616
+ ],
1617
+ "float16"
1618
+ ],
1619
+ [
1620
+ "past_key_values_8",
1621
+ [
1622
+ 1,
1623
+ 8,
1624
+ 16384,
1625
+ 128
1626
+ ],
1627
+ "float16"
1628
+ ],
1629
+ [
1630
+ "past_key_values_9",
1631
+ [
1632
+ 1,
1633
+ 8,
1634
+ 16384,
1635
+ 128
1636
+ ],
1637
+ "float16"
1638
+ ],
1639
+ [
1640
+ "past_key_values_10",
1641
+ [
1642
+ 1,
1643
+ 8,
1644
+ 16384,
1645
+ 128
1646
+ ],
1647
+ "float16"
1648
+ ],
1649
+ [
1650
+ "past_key_values_11",
1651
+ [
1652
+ 1,
1653
+ 8,
1654
+ 16384,
1655
+ 128
1656
+ ],
1657
+ "float16"
1658
+ ],
1659
+ [
1660
+ "past_key_values_12",
1661
+ [
1662
+ 1,
1663
+ 8,
1664
+ 16384,
1665
+ 128
1666
+ ],
1667
+ "float16"
1668
+ ],
1669
+ [
1670
+ "past_key_values_13",
1671
+ [
1672
+ 1,
1673
+ 8,
1674
+ 16384,
1675
+ 128
1676
+ ],
1677
+ "float16"
1678
+ ],
1679
+ [
1680
+ "past_key_values_14",
1681
+ [
1682
+ 1,
1683
+ 8,
1684
+ 16384,
1685
+ 128
1686
+ ],
1687
+ "float16"
1688
+ ],
1689
+ [
1690
+ "past_key_values_15",
1691
+ [
1692
+ 1,
1693
+ 8,
1694
+ 16384,
1695
+ 128
1696
+ ],
1697
+ "float16"
1698
+ ],
1699
+ [
1700
+ "past_key_values_16",
1701
+ [
1702
+ 1,
1703
+ 8,
1704
+ 16384,
1705
+ 128
1706
+ ],
1707
+ "float16"
1708
+ ],
1709
+ [
1710
+ "past_key_values_17",
1711
+ [
1712
+ 1,
1713
+ 8,
1714
+ 16384,
1715
+ 128
1716
+ ],
1717
+ "float16"
1718
+ ],
1719
+ [
1720
+ "past_key_values_18",
1721
+ [
1722
+ 1,
1723
+ 8,
1724
+ 16384,
1725
+ 128
1726
+ ],
1727
+ "float16"
1728
+ ],
1729
+ [
1730
+ "past_key_values_19",
1731
+ [
1732
+ 1,
1733
+ 8,
1734
+ 16384,
1735
+ 128
1736
+ ],
1737
+ "float16"
1738
+ ],
1739
+ [
1740
+ "past_key_values_20",
1741
+ [
1742
+ 1,
1743
+ 8,
1744
+ 16384,
1745
+ 128
1746
+ ],
1747
+ "float16"
1748
+ ],
1749
+ [
1750
+ "past_key_values_21",
1751
+ [
1752
+ 1,
1753
+ 8,
1754
+ 16384,
1755
+ 128
1756
+ ],
1757
+ "float16"
1758
+ ],
1759
+ [
1760
+ "past_key_values_22",
1761
+ [
1762
+ 1,
1763
+ 8,
1764
+ 16384,
1765
+ 128
1766
+ ],
1767
+ "float16"
1768
+ ],
1769
+ [
1770
+ "past_key_values_23",
1771
+ [
1772
+ 1,
1773
+ 8,
1774
+ 16384,
1775
+ 128
1776
+ ],
1777
+ "float16"
1778
+ ],
1779
+ [
1780
+ "past_key_values_24",
1781
+ [
1782
+ 1,
1783
+ 8,
1784
+ 16384,
1785
+ 128
1786
+ ],
1787
+ "float16"
1788
+ ],
1789
+ [
1790
+ "past_key_values_25",
1791
+ [
1792
+ 1,
1793
+ 8,
1794
+ 16384,
1795
+ 128
1796
+ ],
1797
+ "float16"
1798
+ ],
1799
+ [
1800
+ "past_key_values_26",
1801
+ [
1802
+ 1,
1803
+ 8,
1804
+ 16384,
1805
+ 128
1806
+ ],
1807
+ "float16"
1808
+ ],
1809
+ [
1810
+ "past_key_values_27",
1811
+ [
1812
+ 1,
1813
+ 8,
1814
+ 16384,
1815
+ 128
1816
+ ],
1817
+ "float16"
1818
+ ],
1819
+ [
1820
+ "past_key_values_28",
1821
+ [
1822
+ 1,
1823
+ 8,
1824
+ 16384,
1825
+ 128
1826
+ ],
1827
+ "float16"
1828
+ ],
1829
+ [
1830
+ "past_key_values_29",
1831
+ [
1832
+ 1,
1833
+ 8,
1834
+ 16384,
1835
+ 128
1836
+ ],
1837
+ "float16"
1838
+ ],
1839
+ [
1840
+ "past_key_values_30",
1841
+ [
1842
+ 1,
1843
+ 8,
1844
+ 16384,
1845
+ 128
1846
+ ],
1847
+ "float16"
1848
+ ],
1849
+ [
1850
+ "past_key_values_31",
1851
+ [
1852
+ 1,
1853
+ 8,
1854
+ 16384,
1855
+ 128
1856
+ ],
1857
+ "float16"
1858
+ ],
1859
+ [
1860
+ "past_key_values_32",
1861
+ [
1862
+ 1,
1863
+ 8,
1864
+ 16384,
1865
+ 128
1866
+ ],
1867
+ "float16"
1868
+ ],
1869
+ [
1870
+ "past_key_values_33",
1871
+ [
1872
+ 1,
1873
+ 8,
1874
+ 16384,
1875
+ 128
1876
+ ],
1877
+ "float16"
1878
+ ],
1879
+ [
1880
+ "past_key_values_34",
1881
+ [
1882
+ 1,
1883
+ 8,
1884
+ 16384,
1885
+ 128
1886
+ ],
1887
+ "float16"
1888
+ ],
1889
+ [
1890
+ "past_key_values_35",
1891
+ [
1892
+ 1,
1893
+ 8,
1894
+ 16384,
1895
+ 128
1896
+ ],
1897
+ "float16"
1898
+ ],
1899
+ [
1900
+ "past_key_values_36",
1901
+ [
1902
+ 1,
1903
+ 8,
1904
+ 16384,
1905
+ 128
1906
+ ],
1907
+ "float16"
1908
+ ],
1909
+ [
1910
+ "past_key_values_37",
1911
+ [
1912
+ 1,
1913
+ 8,
1914
+ 16384,
1915
+ 128
1916
+ ],
1917
+ "float16"
1918
+ ],
1919
+ [
1920
+ "past_key_values_38",
1921
+ [
1922
+ 1,
1923
+ 8,
1924
+ 16384,
1925
+ 128
1926
+ ],
1927
+ "float16"
1928
+ ],
1929
+ [
1930
+ "past_key_values_39",
1931
+ [
1932
+ 1,
1933
+ 8,
1934
+ 16384,
1935
+ 128
1936
+ ],
1937
+ "float16"
1938
+ ],
1939
+ [
1940
+ "past_key_values_40",
1941
+ [
1942
+ 1,
1943
+ 8,
1944
+ 16384,
1945
+ 128
1946
+ ],
1947
+ "float16"
1948
+ ],
1949
+ [
1950
+ "past_key_values_41",
1951
+ [
1952
+ 1,
1953
+ 8,
1954
+ 16384,
1955
+ 128
1956
+ ],
1957
+ "float16"
1958
+ ],
1959
+ [
1960
+ "past_key_values_42",
1961
+ [
1962
+ 1,
1963
+ 8,
1964
+ 16384,
1965
+ 128
1966
+ ],
1967
+ "float16"
1968
+ ],
1969
+ [
1970
+ "past_key_values_43",
1971
+ [
1972
+ 1,
1973
+ 8,
1974
+ 16384,
1975
+ 128
1976
+ ],
1977
+ "float16"
1978
+ ],
1979
+ [
1980
+ "past_key_values_44",
1981
+ [
1982
+ 1,
1983
+ 8,
1984
+ 16384,
1985
+ 128
1986
+ ],
1987
+ "float16"
1988
+ ],
1989
+ [
1990
+ "past_key_values_45",
1991
+ [
1992
+ 1,
1993
+ 8,
1994
+ 16384,
1995
+ 128
1996
+ ],
1997
+ "float16"
1998
+ ],
1999
+ [
2000
+ "past_key_values_46",
2001
+ [
2002
+ 1,
2003
+ 8,
2004
+ 16384,
2005
+ 128
2006
+ ],
2007
+ "float16"
2008
+ ],
2009
+ [
2010
+ "past_key_values_47",
2011
+ [
2012
+ 1,
2013
+ 8,
2014
+ 16384,
2015
+ 128
2016
+ ],
2017
+ "float16"
2018
+ ],
2019
+ [
2020
+ "past_key_values_48",
2021
+ [
2022
+ 1,
2023
+ 8,
2024
+ 16384,
2025
+ 128
2026
+ ],
2027
+ "float16"
2028
+ ],
2029
+ [
2030
+ "past_key_values_49",
2031
+ [
2032
+ 1,
2033
+ 8,
2034
+ 16384,
2035
+ 128
2036
+ ],
2037
+ "float16"
2038
+ ],
2039
+ [
2040
+ "past_key_values_50",
2041
+ [
2042
+ 1,
2043
+ 8,
2044
+ 16384,
2045
+ 128
2046
+ ],
2047
+ "float16"
2048
+ ],
2049
+ [
2050
+ "past_key_values_51",
2051
+ [
2052
+ 1,
2053
+ 8,
2054
+ 16384,
2055
+ 128
2056
+ ],
2057
+ "float16"
2058
+ ],
2059
+ [
2060
+ "past_key_values_52",
2061
+ [
2062
+ 1,
2063
+ 8,
2064
+ 16384,
2065
+ 128
2066
+ ],
2067
+ "float16"
2068
+ ],
2069
+ [
2070
+ "past_key_values_53",
2071
+ [
2072
+ 1,
2073
+ 8,
2074
+ 16384,
2075
+ 128
2076
+ ],
2077
+ "float16"
2078
+ ],
2079
+ [
2080
+ "past_key_values_54",
2081
+ [
2082
+ 1,
2083
+ 8,
2084
+ 16384,
2085
+ 128
2086
+ ],
2087
+ "float16"
2088
+ ],
2089
+ [
2090
+ "past_key_values_55",
2091
+ [
2092
+ 1,
2093
+ 8,
2094
+ 16384,
2095
+ 128
2096
+ ],
2097
+ "float16"
2098
+ ],
2099
+ [
2100
+ "past_key_values_56",
2101
+ [
2102
+ 1,
2103
+ 8,
2104
+ 16384,
2105
+ 128
2106
+ ],
2107
+ "float16"
2108
+ ],
2109
+ [
2110
+ "past_key_values_57",
2111
+ [
2112
+ 1,
2113
+ 8,
2114
+ 16384,
2115
+ 128
2116
+ ],
2117
+ "float16"
2118
+ ],
2119
+ [
2120
+ "past_key_values_58",
2121
+ [
2122
+ 1,
2123
+ 8,
2124
+ 16384,
2125
+ 128
2126
+ ],
2127
+ "float16"
2128
+ ],
2129
+ [
2130
+ "past_key_values_59",
2131
+ [
2132
+ 1,
2133
+ 8,
2134
+ 16384,
2135
+ 128
2136
+ ],
2137
+ "float16"
2138
+ ],
2139
+ [
2140
+ "past_key_values_60",
2141
+ [
2142
+ 1,
2143
+ 8,
2144
+ 16384,
2145
+ 128
2146
+ ],
2147
+ "float16"
2148
+ ],
2149
+ [
2150
+ "past_key_values_61",
2151
+ [
2152
+ 1,
2153
+ 8,
2154
+ 16384,
2155
+ 128
2156
+ ],
2157
+ "float16"
2158
+ ],
2159
+ [
2160
+ "past_key_values_62",
2161
+ [
2162
+ 1,
2163
+ 8,
2164
+ 16384,
2165
+ 128
2166
+ ],
2167
+ "float16"
2168
+ ],
2169
+ [
2170
+ "past_key_values_63",
2171
+ [
2172
+ 1,
2173
+ 8,
2174
+ 16384,
2175
+ 128
2176
+ ],
2177
+ "float16"
2178
+ ],
2179
+ [
2180
+ "past_key_values_64",
2181
+ [
2182
+ 1,
2183
+ 8,
2184
+ 16384,
2185
+ 128
2186
+ ],
2187
+ "float16"
2188
+ ],
2189
+ [
2190
+ "past_key_values_65",
2191
+ [
2192
+ 1,
2193
+ 8,
2194
+ 16384,
2195
+ 128
2196
+ ],
2197
+ "float16"
2198
+ ],
2199
+ [
2200
+ "past_key_values_66",
2201
+ [
2202
+ 1,
2203
+ 8,
2204
+ 16384,
2205
+ 128
2206
+ ],
2207
+ "float16"
2208
+ ],
2209
+ [
2210
+ "past_key_values_67",
2211
+ [
2212
+ 1,
2213
+ 8,
2214
+ 16384,
2215
+ 128
2216
+ ],
2217
+ "float16"
2218
+ ],
2219
+ [
2220
+ "past_key_values_68",
2221
+ [
2222
+ 1,
2223
+ 8,
2224
+ 16384,
2225
+ 128
2226
+ ],
2227
+ "float16"
2228
+ ],
2229
+ [
2230
+ "past_key_values_69",
2231
+ [
2232
+ 1,
2233
+ 8,
2234
+ 16384,
2235
+ 128
2236
+ ],
2237
+ "float16"
2238
+ ],
2239
+ [
2240
+ "past_key_values_70",
2241
+ [
2242
+ 1,
2243
+ 8,
2244
+ 16384,
2245
+ 128
2246
+ ],
2247
+ "float16"
2248
+ ],
2249
+ [
2250
+ "past_key_values_71",
2251
+ [
2252
+ 1,
2253
+ 8,
2254
+ 16384,
2255
+ 128
2256
+ ],
2257
+ "float16"
2258
+ ]
2259
+ ],
2260
+ "npu": "RBLN-CA25",
2261
+ "tensor_parallel_size": 4
2262
+ },
2263
+ {
2264
+ "compiled_model_name": "decoder_batch_4",
2265
+ "input_info": [
2266
+ [
2267
+ "input_ids",
2268
+ [
2269
+ 4,
2270
+ 1
2271
+ ],
2272
+ "int64"
2273
+ ],
2274
+ [
2275
+ "cache_position",
2276
+ [
2277
+ 4,
2278
+ 1
2279
+ ],
2280
+ "int32"
2281
+ ],
2282
+ [
2283
+ "block_tables",
2284
+ [
2285
+ 4,
2286
+ 2
2287
+ ],
2288
+ "int16"
2289
+ ],
2290
+ [
2291
+ "past_key_values_0",
2292
+ [
2293
+ 1,
2294
+ 8,
2295
+ 16384,
2296
+ 128
2297
+ ],
2298
+ "float16"
2299
+ ],
2300
+ [
2301
+ "past_key_values_1",
2302
+ [
2303
+ 1,
2304
+ 8,
2305
+ 16384,
2306
+ 128
2307
+ ],
2308
+ "float16"
2309
+ ],
2310
+ [
2311
+ "past_key_values_2",
2312
+ [
2313
+ 1,
2314
+ 8,
2315
+ 16384,
2316
+ 128
2317
+ ],
2318
+ "float16"
2319
+ ],
2320
+ [
2321
+ "past_key_values_3",
2322
+ [
2323
+ 1,
2324
+ 8,
2325
+ 16384,
2326
+ 128
2327
+ ],
2328
+ "float16"
2329
+ ],
2330
+ [
2331
+ "past_key_values_4",
2332
+ [
2333
+ 1,
2334
+ 8,
2335
+ 16384,
2336
+ 128
2337
+ ],
2338
+ "float16"
2339
+ ],
2340
+ [
2341
+ "past_key_values_5",
2342
+ [
2343
+ 1,
2344
+ 8,
2345
+ 16384,
2346
+ 128
2347
+ ],
2348
+ "float16"
2349
+ ],
2350
+ [
2351
+ "past_key_values_6",
2352
+ [
2353
+ 1,
2354
+ 8,
2355
+ 16384,
2356
+ 128
2357
+ ],
2358
+ "float16"
2359
+ ],
2360
+ [
2361
+ "past_key_values_7",
2362
+ [
2363
+ 1,
2364
+ 8,
2365
+ 16384,
2366
+ 128
2367
+ ],
2368
+ "float16"
2369
+ ],
2370
+ [
2371
+ "past_key_values_8",
2372
+ [
2373
+ 1,
2374
+ 8,
2375
+ 16384,
2376
+ 128
2377
+ ],
2378
+ "float16"
2379
+ ],
2380
+ [
2381
+ "past_key_values_9",
2382
+ [
2383
+ 1,
2384
+ 8,
2385
+ 16384,
2386
+ 128
2387
+ ],
2388
+ "float16"
2389
+ ],
2390
+ [
2391
+ "past_key_values_10",
2392
+ [
2393
+ 1,
2394
+ 8,
2395
+ 16384,
2396
+ 128
2397
+ ],
2398
+ "float16"
2399
+ ],
2400
+ [
2401
+ "past_key_values_11",
2402
+ [
2403
+ 1,
2404
+ 8,
2405
+ 16384,
2406
+ 128
2407
+ ],
2408
+ "float16"
2409
+ ],
2410
+ [
2411
+ "past_key_values_12",
2412
+ [
2413
+ 1,
2414
+ 8,
2415
+ 16384,
2416
+ 128
2417
+ ],
2418
+ "float16"
2419
+ ],
2420
+ [
2421
+ "past_key_values_13",
2422
+ [
2423
+ 1,
2424
+ 8,
2425
+ 16384,
2426
+ 128
2427
+ ],
2428
+ "float16"
2429
+ ],
2430
+ [
2431
+ "past_key_values_14",
2432
+ [
2433
+ 1,
2434
+ 8,
2435
+ 16384,
2436
+ 128
2437
+ ],
2438
+ "float16"
2439
+ ],
2440
+ [
2441
+ "past_key_values_15",
2442
+ [
2443
+ 1,
2444
+ 8,
2445
+ 16384,
2446
+ 128
2447
+ ],
2448
+ "float16"
2449
+ ],
2450
+ [
2451
+ "past_key_values_16",
2452
+ [
2453
+ 1,
2454
+ 8,
2455
+ 16384,
2456
+ 128
2457
+ ],
2458
+ "float16"
2459
+ ],
2460
+ [
2461
+ "past_key_values_17",
2462
+ [
2463
+ 1,
2464
+ 8,
2465
+ 16384,
2466
+ 128
2467
+ ],
2468
+ "float16"
2469
+ ],
2470
+ [
2471
+ "past_key_values_18",
2472
+ [
2473
+ 1,
2474
+ 8,
2475
+ 16384,
2476
+ 128
2477
+ ],
2478
+ "float16"
2479
+ ],
2480
+ [
2481
+ "past_key_values_19",
2482
+ [
2483
+ 1,
2484
+ 8,
2485
+ 16384,
2486
+ 128
2487
+ ],
2488
+ "float16"
2489
+ ],
2490
+ [
2491
+ "past_key_values_20",
2492
+ [
2493
+ 1,
2494
+ 8,
2495
+ 16384,
2496
+ 128
2497
+ ],
2498
+ "float16"
2499
+ ],
2500
+ [
2501
+ "past_key_values_21",
2502
+ [
2503
+ 1,
2504
+ 8,
2505
+ 16384,
2506
+ 128
2507
+ ],
2508
+ "float16"
2509
+ ],
2510
+ [
2511
+ "past_key_values_22",
2512
+ [
2513
+ 1,
2514
+ 8,
2515
+ 16384,
2516
+ 128
2517
+ ],
2518
+ "float16"
2519
+ ],
2520
+ [
2521
+ "past_key_values_23",
2522
+ [
2523
+ 1,
2524
+ 8,
2525
+ 16384,
2526
+ 128
2527
+ ],
2528
+ "float16"
2529
+ ],
2530
+ [
2531
+ "past_key_values_24",
2532
+ [
2533
+ 1,
2534
+ 8,
2535
+ 16384,
2536
+ 128
2537
+ ],
2538
+ "float16"
2539
+ ],
2540
+ [
2541
+ "past_key_values_25",
2542
+ [
2543
+ 1,
2544
+ 8,
2545
+ 16384,
2546
+ 128
2547
+ ],
2548
+ "float16"
2549
+ ],
2550
+ [
2551
+ "past_key_values_26",
2552
+ [
2553
+ 1,
2554
+ 8,
2555
+ 16384,
2556
+ 128
2557
+ ],
2558
+ "float16"
2559
+ ],
2560
+ [
2561
+ "past_key_values_27",
2562
+ [
2563
+ 1,
2564
+ 8,
2565
+ 16384,
2566
+ 128
2567
+ ],
2568
+ "float16"
2569
+ ],
2570
+ [
2571
+ "past_key_values_28",
2572
+ [
2573
+ 1,
2574
+ 8,
2575
+ 16384,
2576
+ 128
2577
+ ],
2578
+ "float16"
2579
+ ],
2580
+ [
2581
+ "past_key_values_29",
2582
+ [
2583
+ 1,
2584
+ 8,
2585
+ 16384,
2586
+ 128
2587
+ ],
2588
+ "float16"
2589
+ ],
2590
+ [
2591
+ "past_key_values_30",
2592
+ [
2593
+ 1,
2594
+ 8,
2595
+ 16384,
2596
+ 128
2597
+ ],
2598
+ "float16"
2599
+ ],
2600
+ [
2601
+ "past_key_values_31",
2602
+ [
2603
+ 1,
2604
+ 8,
2605
+ 16384,
2606
+ 128
2607
+ ],
2608
+ "float16"
2609
+ ],
2610
+ [
2611
+ "past_key_values_32",
2612
+ [
2613
+ 1,
2614
+ 8,
2615
+ 16384,
2616
+ 128
2617
+ ],
2618
+ "float16"
2619
+ ],
2620
+ [
2621
+ "past_key_values_33",
2622
+ [
2623
+ 1,
2624
+ 8,
2625
+ 16384,
2626
+ 128
2627
+ ],
2628
+ "float16"
2629
+ ],
2630
+ [
2631
+ "past_key_values_34",
2632
+ [
2633
+ 1,
2634
+ 8,
2635
+ 16384,
2636
+ 128
2637
+ ],
2638
+ "float16"
2639
+ ],
2640
+ [
2641
+ "past_key_values_35",
2642
+ [
2643
+ 1,
2644
+ 8,
2645
+ 16384,
2646
+ 128
2647
+ ],
2648
+ "float16"
2649
+ ],
2650
+ [
2651
+ "past_key_values_36",
2652
+ [
2653
+ 1,
2654
+ 8,
2655
+ 16384,
2656
+ 128
2657
+ ],
2658
+ "float16"
2659
+ ],
2660
+ [
2661
+ "past_key_values_37",
2662
+ [
2663
+ 1,
2664
+ 8,
2665
+ 16384,
2666
+ 128
2667
+ ],
2668
+ "float16"
2669
+ ],
2670
+ [
2671
+ "past_key_values_38",
2672
+ [
2673
+ 1,
2674
+ 8,
2675
+ 16384,
2676
+ 128
2677
+ ],
2678
+ "float16"
2679
+ ],
2680
+ [
2681
+ "past_key_values_39",
2682
+ [
2683
+ 1,
2684
+ 8,
2685
+ 16384,
2686
+ 128
2687
+ ],
2688
+ "float16"
2689
+ ],
2690
+ [
2691
+ "past_key_values_40",
2692
+ [
2693
+ 1,
2694
+ 8,
2695
+ 16384,
2696
+ 128
2697
+ ],
2698
+ "float16"
2699
+ ],
2700
+ [
2701
+ "past_key_values_41",
2702
+ [
2703
+ 1,
2704
+ 8,
2705
+ 16384,
2706
+ 128
2707
+ ],
2708
+ "float16"
2709
+ ],
2710
+ [
2711
+ "past_key_values_42",
2712
+ [
2713
+ 1,
2714
+ 8,
2715
+ 16384,
2716
+ 128
2717
+ ],
2718
+ "float16"
2719
+ ],
2720
+ [
2721
+ "past_key_values_43",
2722
+ [
2723
+ 1,
2724
+ 8,
2725
+ 16384,
2726
+ 128
2727
+ ],
2728
+ "float16"
2729
+ ],
2730
+ [
2731
+ "past_key_values_44",
2732
+ [
2733
+ 1,
2734
+ 8,
2735
+ 16384,
2736
+ 128
2737
+ ],
2738
+ "float16"
2739
+ ],
2740
+ [
2741
+ "past_key_values_45",
2742
+ [
2743
+ 1,
2744
+ 8,
2745
+ 16384,
2746
+ 128
2747
+ ],
2748
+ "float16"
2749
+ ],
2750
+ [
2751
+ "past_key_values_46",
2752
+ [
2753
+ 1,
2754
+ 8,
2755
+ 16384,
2756
+ 128
2757
+ ],
2758
+ "float16"
2759
+ ],
2760
+ [
2761
+ "past_key_values_47",
2762
+ [
2763
+ 1,
2764
+ 8,
2765
+ 16384,
2766
+ 128
2767
+ ],
2768
+ "float16"
2769
+ ],
2770
+ [
2771
+ "past_key_values_48",
2772
+ [
2773
+ 1,
2774
+ 8,
2775
+ 16384,
2776
+ 128
2777
+ ],
2778
+ "float16"
2779
+ ],
2780
+ [
2781
+ "past_key_values_49",
2782
+ [
2783
+ 1,
2784
+ 8,
2785
+ 16384,
2786
+ 128
2787
+ ],
2788
+ "float16"
2789
+ ],
2790
+ [
2791
+ "past_key_values_50",
2792
+ [
2793
+ 1,
2794
+ 8,
2795
+ 16384,
2796
+ 128
2797
+ ],
2798
+ "float16"
2799
+ ],
2800
+ [
2801
+ "past_key_values_51",
2802
+ [
2803
+ 1,
2804
+ 8,
2805
+ 16384,
2806
+ 128
2807
+ ],
2808
+ "float16"
2809
+ ],
2810
+ [
2811
+ "past_key_values_52",
2812
+ [
2813
+ 1,
2814
+ 8,
2815
+ 16384,
2816
+ 128
2817
+ ],
2818
+ "float16"
2819
+ ],
2820
+ [
2821
+ "past_key_values_53",
2822
+ [
2823
+ 1,
2824
+ 8,
2825
+ 16384,
2826
+ 128
2827
+ ],
2828
+ "float16"
2829
+ ],
2830
+ [
2831
+ "past_key_values_54",
2832
+ [
2833
+ 1,
2834
+ 8,
2835
+ 16384,
2836
+ 128
2837
+ ],
2838
+ "float16"
2839
+ ],
2840
+ [
2841
+ "past_key_values_55",
2842
+ [
2843
+ 1,
2844
+ 8,
2845
+ 16384,
2846
+ 128
2847
+ ],
2848
+ "float16"
2849
+ ],
2850
+ [
2851
+ "past_key_values_56",
2852
+ [
2853
+ 1,
2854
+ 8,
2855
+ 16384,
2856
+ 128
2857
+ ],
2858
+ "float16"
2859
+ ],
2860
+ [
2861
+ "past_key_values_57",
2862
+ [
2863
+ 1,
2864
+ 8,
2865
+ 16384,
2866
+ 128
2867
+ ],
2868
+ "float16"
2869
+ ],
2870
+ [
2871
+ "past_key_values_58",
2872
+ [
2873
+ 1,
2874
+ 8,
2875
+ 16384,
2876
+ 128
2877
+ ],
2878
+ "float16"
2879
+ ],
2880
+ [
2881
+ "past_key_values_59",
2882
+ [
2883
+ 1,
2884
+ 8,
2885
+ 16384,
2886
+ 128
2887
+ ],
2888
+ "float16"
2889
+ ],
2890
+ [
2891
+ "past_key_values_60",
2892
+ [
2893
+ 1,
2894
+ 8,
2895
+ 16384,
2896
+ 128
2897
+ ],
2898
+ "float16"
2899
+ ],
2900
+ [
2901
+ "past_key_values_61",
2902
+ [
2903
+ 1,
2904
+ 8,
2905
+ 16384,
2906
+ 128
2907
+ ],
2908
+ "float16"
2909
+ ],
2910
+ [
2911
+ "past_key_values_62",
2912
+ [
2913
+ 1,
2914
+ 8,
2915
+ 16384,
2916
+ 128
2917
+ ],
2918
+ "float16"
2919
+ ],
2920
+ [
2921
+ "past_key_values_63",
2922
+ [
2923
+ 1,
2924
+ 8,
2925
+ 16384,
2926
+ 128
2927
+ ],
2928
+ "float16"
2929
+ ],
2930
+ [
2931
+ "past_key_values_64",
2932
+ [
2933
+ 1,
2934
+ 8,
2935
+ 16384,
2936
+ 128
2937
+ ],
2938
+ "float16"
2939
+ ],
2940
+ [
2941
+ "past_key_values_65",
2942
+ [
2943
+ 1,
2944
+ 8,
2945
+ 16384,
2946
+ 128
2947
+ ],
2948
+ "float16"
2949
+ ],
2950
+ [
2951
+ "past_key_values_66",
2952
+ [
2953
+ 1,
2954
+ 8,
2955
+ 16384,
2956
+ 128
2957
+ ],
2958
+ "float16"
2959
+ ],
2960
+ [
2961
+ "past_key_values_67",
2962
+ [
2963
+ 1,
2964
+ 8,
2965
+ 16384,
2966
+ 128
2967
+ ],
2968
+ "float16"
2969
+ ],
2970
+ [
2971
+ "past_key_values_68",
2972
+ [
2973
+ 1,
2974
+ 8,
2975
+ 16384,
2976
+ 128
2977
+ ],
2978
+ "float16"
2979
+ ],
2980
+ [
2981
+ "past_key_values_69",
2982
+ [
2983
+ 1,
2984
+ 8,
2985
+ 16384,
2986
+ 128
2987
+ ],
2988
+ "float16"
2989
+ ],
2990
+ [
2991
+ "past_key_values_70",
2992
+ [
2993
+ 1,
2994
+ 8,
2995
+ 16384,
2996
+ 128
2997
+ ],
2998
+ "float16"
2999
+ ],
3000
+ [
3001
+ "past_key_values_71",
3002
+ [
3003
+ 1,
3004
+ 8,
3005
+ 16384,
3006
+ 128
3007
+ ],
3008
+ "float16"
3009
+ ]
3010
+ ],
3011
+ "npu": "RBLN-CA25",
3012
+ "tensor_parallel_size": 4
3013
+ },
3014
+ {
3015
+ "compiled_model_name": "decoder_batch_1",
3016
+ "input_info": [
3017
+ [
3018
+ "input_ids",
3019
+ [
3020
+ 1,
3021
+ 1
3022
+ ],
3023
+ "int64"
3024
+ ],
3025
+ [
3026
+ "cache_position",
3027
+ [
3028
+ 1,
3029
+ 1
3030
+ ],
3031
+ "int32"
3032
+ ],
3033
+ [
3034
+ "block_tables",
3035
+ [
3036
+ 1,
3037
+ 2
3038
+ ],
3039
+ "int16"
3040
+ ],
3041
+ [
3042
+ "past_key_values_0",
3043
+ [
3044
+ 1,
3045
+ 8,
3046
+ 16384,
3047
+ 128
3048
+ ],
3049
+ "float16"
3050
+ ],
3051
+ [
3052
+ "past_key_values_1",
3053
+ [
3054
+ 1,
3055
+ 8,
3056
+ 16384,
3057
+ 128
3058
+ ],
3059
+ "float16"
3060
+ ],
3061
+ [
3062
+ "past_key_values_2",
3063
+ [
3064
+ 1,
3065
+ 8,
3066
+ 16384,
3067
+ 128
3068
+ ],
3069
+ "float16"
3070
+ ],
3071
+ [
3072
+ "past_key_values_3",
3073
+ [
3074
+ 1,
3075
+ 8,
3076
+ 16384,
3077
+ 128
3078
+ ],
3079
+ "float16"
3080
+ ],
3081
+ [
3082
+ "past_key_values_4",
3083
+ [
3084
+ 1,
3085
+ 8,
3086
+ 16384,
3087
+ 128
3088
+ ],
3089
+ "float16"
3090
+ ],
3091
+ [
3092
+ "past_key_values_5",
3093
+ [
3094
+ 1,
3095
+ 8,
3096
+ 16384,
3097
+ 128
3098
+ ],
3099
+ "float16"
3100
+ ],
3101
+ [
3102
+ "past_key_values_6",
3103
+ [
3104
+ 1,
3105
+ 8,
3106
+ 16384,
3107
+ 128
3108
+ ],
3109
+ "float16"
3110
+ ],
3111
+ [
3112
+ "past_key_values_7",
3113
+ [
3114
+ 1,
3115
+ 8,
3116
+ 16384,
3117
+ 128
3118
+ ],
3119
+ "float16"
3120
+ ],
3121
+ [
3122
+ "past_key_values_8",
3123
+ [
3124
+ 1,
3125
+ 8,
3126
+ 16384,
3127
+ 128
3128
+ ],
3129
+ "float16"
3130
+ ],
3131
+ [
3132
+ "past_key_values_9",
3133
+ [
3134
+ 1,
3135
+ 8,
3136
+ 16384,
3137
+ 128
3138
+ ],
3139
+ "float16"
3140
+ ],
3141
+ [
3142
+ "past_key_values_10",
3143
+ [
3144
+ 1,
3145
+ 8,
3146
+ 16384,
3147
+ 128
3148
+ ],
3149
+ "float16"
3150
+ ],
3151
+ [
3152
+ "past_key_values_11",
3153
+ [
3154
+ 1,
3155
+ 8,
3156
+ 16384,
3157
+ 128
3158
+ ],
3159
+ "float16"
3160
+ ],
3161
+ [
3162
+ "past_key_values_12",
3163
+ [
3164
+ 1,
3165
+ 8,
3166
+ 16384,
3167
+ 128
3168
+ ],
3169
+ "float16"
3170
+ ],
3171
+ [
3172
+ "past_key_values_13",
3173
+ [
3174
+ 1,
3175
+ 8,
3176
+ 16384,
3177
+ 128
3178
+ ],
3179
+ "float16"
3180
+ ],
3181
+ [
3182
+ "past_key_values_14",
3183
+ [
3184
+ 1,
3185
+ 8,
3186
+ 16384,
3187
+ 128
3188
+ ],
3189
+ "float16"
3190
+ ],
3191
+ [
3192
+ "past_key_values_15",
3193
+ [
3194
+ 1,
3195
+ 8,
3196
+ 16384,
3197
+ 128
3198
+ ],
3199
+ "float16"
3200
+ ],
3201
+ [
3202
+ "past_key_values_16",
3203
+ [
3204
+ 1,
3205
+ 8,
3206
+ 16384,
3207
+ 128
3208
+ ],
3209
+ "float16"
3210
+ ],
3211
+ [
3212
+ "past_key_values_17",
3213
+ [
3214
+ 1,
3215
+ 8,
3216
+ 16384,
3217
+ 128
3218
+ ],
3219
+ "float16"
3220
+ ],
3221
+ [
3222
+ "past_key_values_18",
3223
+ [
3224
+ 1,
3225
+ 8,
3226
+ 16384,
3227
+ 128
3228
+ ],
3229
+ "float16"
3230
+ ],
3231
+ [
3232
+ "past_key_values_19",
3233
+ [
3234
+ 1,
3235
+ 8,
3236
+ 16384,
3237
+ 128
3238
+ ],
3239
+ "float16"
3240
+ ],
3241
+ [
3242
+ "past_key_values_20",
3243
+ [
3244
+ 1,
3245
+ 8,
3246
+ 16384,
3247
+ 128
3248
+ ],
3249
+ "float16"
3250
+ ],
3251
+ [
3252
+ "past_key_values_21",
3253
+ [
3254
+ 1,
3255
+ 8,
3256
+ 16384,
3257
+ 128
3258
+ ],
3259
+ "float16"
3260
+ ],
3261
+ [
3262
+ "past_key_values_22",
3263
+ [
3264
+ 1,
3265
+ 8,
3266
+ 16384,
3267
+ 128
3268
+ ],
3269
+ "float16"
3270
+ ],
3271
+ [
3272
+ "past_key_values_23",
3273
+ [
3274
+ 1,
3275
+ 8,
3276
+ 16384,
3277
+ 128
3278
+ ],
3279
+ "float16"
3280
+ ],
3281
+ [
3282
+ "past_key_values_24",
3283
+ [
3284
+ 1,
3285
+ 8,
3286
+ 16384,
3287
+ 128
3288
+ ],
3289
+ "float16"
3290
+ ],
3291
+ [
3292
+ "past_key_values_25",
3293
+ [
3294
+ 1,
3295
+ 8,
3296
+ 16384,
3297
+ 128
3298
+ ],
3299
+ "float16"
3300
+ ],
3301
+ [
3302
+ "past_key_values_26",
3303
+ [
3304
+ 1,
3305
+ 8,
3306
+ 16384,
3307
+ 128
3308
+ ],
3309
+ "float16"
3310
+ ],
3311
+ [
3312
+ "past_key_values_27",
3313
+ [
3314
+ 1,
3315
+ 8,
3316
+ 16384,
3317
+ 128
3318
+ ],
3319
+ "float16"
3320
+ ],
3321
+ [
3322
+ "past_key_values_28",
3323
+ [
3324
+ 1,
3325
+ 8,
3326
+ 16384,
3327
+ 128
3328
+ ],
3329
+ "float16"
3330
+ ],
3331
+ [
3332
+ "past_key_values_29",
3333
+ [
3334
+ 1,
3335
+ 8,
3336
+ 16384,
3337
+ 128
3338
+ ],
3339
+ "float16"
3340
+ ],
3341
+ [
3342
+ "past_key_values_30",
3343
+ [
3344
+ 1,
3345
+ 8,
3346
+ 16384,
3347
+ 128
3348
+ ],
3349
+ "float16"
3350
+ ],
3351
+ [
3352
+ "past_key_values_31",
3353
+ [
3354
+ 1,
3355
+ 8,
3356
+ 16384,
3357
+ 128
3358
+ ],
3359
+ "float16"
3360
+ ],
3361
+ [
3362
+ "past_key_values_32",
3363
+ [
3364
+ 1,
3365
+ 8,
3366
+ 16384,
3367
+ 128
3368
+ ],
3369
+ "float16"
3370
+ ],
3371
+ [
3372
+ "past_key_values_33",
3373
+ [
3374
+ 1,
3375
+ 8,
3376
+ 16384,
3377
+ 128
3378
+ ],
3379
+ "float16"
3380
+ ],
3381
+ [
3382
+ "past_key_values_34",
3383
+ [
3384
+ 1,
3385
+ 8,
3386
+ 16384,
3387
+ 128
3388
+ ],
3389
+ "float16"
3390
+ ],
3391
+ [
3392
+ "past_key_values_35",
3393
+ [
3394
+ 1,
3395
+ 8,
3396
+ 16384,
3397
+ 128
3398
+ ],
3399
+ "float16"
3400
+ ],
3401
+ [
3402
+ "past_key_values_36",
3403
+ [
3404
+ 1,
3405
+ 8,
3406
+ 16384,
3407
+ 128
3408
+ ],
3409
+ "float16"
3410
+ ],
3411
+ [
3412
+ "past_key_values_37",
3413
+ [
3414
+ 1,
3415
+ 8,
3416
+ 16384,
3417
+ 128
3418
+ ],
3419
+ "float16"
3420
+ ],
3421
+ [
3422
+ "past_key_values_38",
3423
+ [
3424
+ 1,
3425
+ 8,
3426
+ 16384,
3427
+ 128
3428
+ ],
3429
+ "float16"
3430
+ ],
3431
+ [
3432
+ "past_key_values_39",
3433
+ [
3434
+ 1,
3435
+ 8,
3436
+ 16384,
3437
+ 128
3438
+ ],
3439
+ "float16"
3440
+ ],
3441
+ [
3442
+ "past_key_values_40",
3443
+ [
3444
+ 1,
3445
+ 8,
3446
+ 16384,
3447
+ 128
3448
+ ],
3449
+ "float16"
3450
+ ],
3451
+ [
3452
+ "past_key_values_41",
3453
+ [
3454
+ 1,
3455
+ 8,
3456
+ 16384,
3457
+ 128
3458
+ ],
3459
+ "float16"
3460
+ ],
3461
+ [
3462
+ "past_key_values_42",
3463
+ [
3464
+ 1,
3465
+ 8,
3466
+ 16384,
3467
+ 128
3468
+ ],
3469
+ "float16"
3470
+ ],
3471
+ [
3472
+ "past_key_values_43",
3473
+ [
3474
+ 1,
3475
+ 8,
3476
+ 16384,
3477
+ 128
3478
+ ],
3479
+ "float16"
3480
+ ],
3481
+ [
3482
+ "past_key_values_44",
3483
+ [
3484
+ 1,
3485
+ 8,
3486
+ 16384,
3487
+ 128
3488
+ ],
3489
+ "float16"
3490
+ ],
3491
+ [
3492
+ "past_key_values_45",
3493
+ [
3494
+ 1,
3495
+ 8,
3496
+ 16384,
3497
+ 128
3498
+ ],
3499
+ "float16"
3500
+ ],
3501
+ [
3502
+ "past_key_values_46",
3503
+ [
3504
+ 1,
3505
+ 8,
3506
+ 16384,
3507
+ 128
3508
+ ],
3509
+ "float16"
3510
+ ],
3511
+ [
3512
+ "past_key_values_47",
3513
+ [
3514
+ 1,
3515
+ 8,
3516
+ 16384,
3517
+ 128
3518
+ ],
3519
+ "float16"
3520
+ ],
3521
+ [
3522
+ "past_key_values_48",
3523
+ [
3524
+ 1,
3525
+ 8,
3526
+ 16384,
3527
+ 128
3528
+ ],
3529
+ "float16"
3530
+ ],
3531
+ [
3532
+ "past_key_values_49",
3533
+ [
3534
+ 1,
3535
+ 8,
3536
+ 16384,
3537
+ 128
3538
+ ],
3539
+ "float16"
3540
+ ],
3541
+ [
3542
+ "past_key_values_50",
3543
+ [
3544
+ 1,
3545
+ 8,
3546
+ 16384,
3547
+ 128
3548
+ ],
3549
+ "float16"
3550
+ ],
3551
+ [
3552
+ "past_key_values_51",
3553
+ [
3554
+ 1,
3555
+ 8,
3556
+ 16384,
3557
+ 128
3558
+ ],
3559
+ "float16"
3560
+ ],
3561
+ [
3562
+ "past_key_values_52",
3563
+ [
3564
+ 1,
3565
+ 8,
3566
+ 16384,
3567
+ 128
3568
+ ],
3569
+ "float16"
3570
+ ],
3571
+ [
3572
+ "past_key_values_53",
3573
+ [
3574
+ 1,
3575
+ 8,
3576
+ 16384,
3577
+ 128
3578
+ ],
3579
+ "float16"
3580
+ ],
3581
+ [
3582
+ "past_key_values_54",
3583
+ [
3584
+ 1,
3585
+ 8,
3586
+ 16384,
3587
+ 128
3588
+ ],
3589
+ "float16"
3590
+ ],
3591
+ [
3592
+ "past_key_values_55",
3593
+ [
3594
+ 1,
3595
+ 8,
3596
+ 16384,
3597
+ 128
3598
+ ],
3599
+ "float16"
3600
+ ],
3601
+ [
3602
+ "past_key_values_56",
3603
+ [
3604
+ 1,
3605
+ 8,
3606
+ 16384,
3607
+ 128
3608
+ ],
3609
+ "float16"
3610
+ ],
3611
+ [
3612
+ "past_key_values_57",
3613
+ [
3614
+ 1,
3615
+ 8,
3616
+ 16384,
3617
+ 128
3618
+ ],
3619
+ "float16"
3620
+ ],
3621
+ [
3622
+ "past_key_values_58",
3623
+ [
3624
+ 1,
3625
+ 8,
3626
+ 16384,
3627
+ 128
3628
+ ],
3629
+ "float16"
3630
+ ],
3631
+ [
3632
+ "past_key_values_59",
3633
+ [
3634
+ 1,
3635
+ 8,
3636
+ 16384,
3637
+ 128
3638
+ ],
3639
+ "float16"
3640
+ ],
3641
+ [
3642
+ "past_key_values_60",
3643
+ [
3644
+ 1,
3645
+ 8,
3646
+ 16384,
3647
+ 128
3648
+ ],
3649
+ "float16"
3650
+ ],
3651
+ [
3652
+ "past_key_values_61",
3653
+ [
3654
+ 1,
3655
+ 8,
3656
+ 16384,
3657
+ 128
3658
+ ],
3659
+ "float16"
3660
+ ],
3661
+ [
3662
+ "past_key_values_62",
3663
+ [
3664
+ 1,
3665
+ 8,
3666
+ 16384,
3667
+ 128
3668
+ ],
3669
+ "float16"
3670
+ ],
3671
+ [
3672
+ "past_key_values_63",
3673
+ [
3674
+ 1,
3675
+ 8,
3676
+ 16384,
3677
+ 128
3678
+ ],
3679
+ "float16"
3680
+ ],
3681
+ [
3682
+ "past_key_values_64",
3683
+ [
3684
+ 1,
3685
+ 8,
3686
+ 16384,
3687
+ 128
3688
+ ],
3689
+ "float16"
3690
+ ],
3691
+ [
3692
+ "past_key_values_65",
3693
+ [
3694
+ 1,
3695
+ 8,
3696
+ 16384,
3697
+ 128
3698
+ ],
3699
+ "float16"
3700
+ ],
3701
+ [
3702
+ "past_key_values_66",
3703
+ [
3704
+ 1,
3705
+ 8,
3706
+ 16384,
3707
+ 128
3708
+ ],
3709
+ "float16"
3710
+ ],
3711
+ [
3712
+ "past_key_values_67",
3713
+ [
3714
+ 1,
3715
+ 8,
3716
+ 16384,
3717
+ 128
3718
+ ],
3719
+ "float16"
3720
+ ],
3721
+ [
3722
+ "past_key_values_68",
3723
+ [
3724
+ 1,
3725
+ 8,
3726
+ 16384,
3727
+ 128
3728
+ ],
3729
+ "float16"
3730
+ ],
3731
+ [
3732
+ "past_key_values_69",
3733
+ [
3734
+ 1,
3735
+ 8,
3736
+ 16384,
3737
+ 128
3738
+ ],
3739
+ "float16"
3740
+ ],
3741
+ [
3742
+ "past_key_values_70",
3743
+ [
3744
+ 1,
3745
+ 8,
3746
+ 16384,
3747
+ 128
3748
+ ],
3749
+ "float16"
3750
+ ],
3751
+ [
3752
+ "past_key_values_71",
3753
+ [
3754
+ 1,
3755
+ 8,
3756
+ 16384,
3757
+ 128
3758
+ ],
3759
+ "float16"
3760
+ ]
3761
+ ],
3762
+ "npu": "RBLN-CA25",
3763
+ "tensor_parallel_size": 4
3764
+ }
3765
+ ],
3766
+ "batch_size": 12,
3767
+ "max_seq_len": 32768,
3768
+ "use_inputs_embeds": false,
3769
+ "use_position_ids": false,
3770
+ "use_attention_mask": false,
3771
+ "quantization": {
3772
+ "format": "rbln",
3773
+ "weights": "int8",
3774
+ "activations": "fp16",
3775
+ "kv_caches": "fp16"
3776
+ },
3777
+ "lora_config": null,
3778
+ "attn_impl": "flash_attn",
3779
+ "kvcache_partition_len": 16384,
3780
+ "kvcache_block_size": 16384,
3781
+ "prefill_chunk_size": 128,
3782
+ "kvcache_num_blocks": 24,
3783
+ "cache_impl": "static",
3784
+ "sliding_window": null,
3785
+ "sliding_window_layers": [],
3786
+ "phases": [
3787
+ "prefill",
3788
+ "decode"
3789
+ ],
3790
+ "logits_to_keep": 1,
3791
+ "output_hidden_states": false,
3792
+ "decoder_batch_sizes": [
3793
+ 12,
3794
+ 8,
3795
+ 4,
3796
+ 1
3797
+ ],
3798
+ "kvcache_metas": [
3799
+ {
3800
+ "name": "past_key_values_0",
3801
+ "layer_index": 0,
3802
+ "shape": [
3803
+ 24,
3804
+ 8,
3805
+ 16384,
3806
+ 128
3807
+ ],
3808
+ "layer_type": "full_attention",
3809
+ "is_auto": true,
3810
+ "dtype": "float16"
3811
+ },
3812
+ {
3813
+ "name": "past_key_values_1",
3814
+ "layer_index": 0,
3815
+ "shape": [
3816
+ 24,
3817
+ 8,
3818
+ 16384,
3819
+ 128
3820
+ ],
3821
+ "layer_type": "full_attention",
3822
+ "is_auto": true,
3823
+ "dtype": "float16"
3824
+ },
3825
+ {
3826
+ "name": "past_key_values_2",
3827
+ "layer_index": 1,
3828
+ "shape": [
3829
+ 24,
3830
+ 8,
3831
+ 16384,
3832
+ 128
3833
+ ],
3834
+ "layer_type": "full_attention",
3835
+ "is_auto": true,
3836
+ "dtype": "float16"
3837
+ },
3838
+ {
3839
+ "name": "past_key_values_3",
3840
+ "layer_index": 1,
3841
+ "shape": [
3842
+ 24,
3843
+ 8,
3844
+ 16384,
3845
+ 128
3846
+ ],
3847
+ "layer_type": "full_attention",
3848
+ "is_auto": true,
3849
+ "dtype": "float16"
3850
+ },
3851
+ {
3852
+ "name": "past_key_values_4",
3853
+ "layer_index": 2,
3854
+ "shape": [
3855
+ 24,
3856
+ 8,
3857
+ 16384,
3858
+ 128
3859
+ ],
3860
+ "layer_type": "full_attention",
3861
+ "is_auto": true,
3862
+ "dtype": "float16"
3863
+ },
3864
+ {
3865
+ "name": "past_key_values_5",
3866
+ "layer_index": 2,
3867
+ "shape": [
3868
+ 24,
3869
+ 8,
3870
+ 16384,
3871
+ 128
3872
+ ],
3873
+ "layer_type": "full_attention",
3874
+ "is_auto": true,
3875
+ "dtype": "float16"
3876
+ },
3877
+ {
3878
+ "name": "past_key_values_6",
3879
+ "layer_index": 3,
3880
+ "shape": [
3881
+ 24,
3882
+ 8,
3883
+ 16384,
3884
+ 128
3885
+ ],
3886
+ "layer_type": "full_attention",
3887
+ "is_auto": true,
3888
+ "dtype": "float16"
3889
+ },
3890
+ {
3891
+ "name": "past_key_values_7",
3892
+ "layer_index": 3,
3893
+ "shape": [
3894
+ 24,
3895
+ 8,
3896
+ 16384,
3897
+ 128
3898
+ ],
3899
+ "layer_type": "full_attention",
3900
+ "is_auto": true,
3901
+ "dtype": "float16"
3902
+ },
3903
+ {
3904
+ "name": "past_key_values_8",
3905
+ "layer_index": 4,
3906
+ "shape": [
3907
+ 24,
3908
+ 8,
3909
+ 16384,
3910
+ 128
3911
+ ],
3912
+ "layer_type": "full_attention",
3913
+ "is_auto": true,
3914
+ "dtype": "float16"
3915
+ },
3916
+ {
3917
+ "name": "past_key_values_9",
3918
+ "layer_index": 4,
3919
+ "shape": [
3920
+ 24,
3921
+ 8,
3922
+ 16384,
3923
+ 128
3924
+ ],
3925
+ "layer_type": "full_attention",
3926
+ "is_auto": true,
3927
+ "dtype": "float16"
3928
+ },
3929
+ {
3930
+ "name": "past_key_values_10",
3931
+ "layer_index": 5,
3932
+ "shape": [
3933
+ 24,
3934
+ 8,
3935
+ 16384,
3936
+ 128
3937
+ ],
3938
+ "layer_type": "full_attention",
3939
+ "is_auto": true,
3940
+ "dtype": "float16"
3941
+ },
3942
+ {
3943
+ "name": "past_key_values_11",
3944
+ "layer_index": 5,
3945
+ "shape": [
3946
+ 24,
3947
+ 8,
3948
+ 16384,
3949
+ 128
3950
+ ],
3951
+ "layer_type": "full_attention",
3952
+ "is_auto": true,
3953
+ "dtype": "float16"
3954
+ },
3955
+ {
3956
+ "name": "past_key_values_12",
3957
+ "layer_index": 6,
3958
+ "shape": [
3959
+ 24,
3960
+ 8,
3961
+ 16384,
3962
+ 128
3963
+ ],
3964
+ "layer_type": "full_attention",
3965
+ "is_auto": true,
3966
+ "dtype": "float16"
3967
+ },
3968
+ {
3969
+ "name": "past_key_values_13",
3970
+ "layer_index": 6,
3971
+ "shape": [
3972
+ 24,
3973
+ 8,
3974
+ 16384,
3975
+ 128
3976
+ ],
3977
+ "layer_type": "full_attention",
3978
+ "is_auto": true,
3979
+ "dtype": "float16"
3980
+ },
3981
+ {
3982
+ "name": "past_key_values_14",
3983
+ "layer_index": 7,
3984
+ "shape": [
3985
+ 24,
3986
+ 8,
3987
+ 16384,
3988
+ 128
3989
+ ],
3990
+ "layer_type": "full_attention",
3991
+ "is_auto": true,
3992
+ "dtype": "float16"
3993
+ },
3994
+ {
3995
+ "name": "past_key_values_15",
3996
+ "layer_index": 7,
3997
+ "shape": [
3998
+ 24,
3999
+ 8,
4000
+ 16384,
4001
+ 128
4002
+ ],
4003
+ "layer_type": "full_attention",
4004
+ "is_auto": true,
4005
+ "dtype": "float16"
4006
+ },
4007
+ {
4008
+ "name": "past_key_values_16",
4009
+ "layer_index": 8,
4010
+ "shape": [
4011
+ 24,
4012
+ 8,
4013
+ 16384,
4014
+ 128
4015
+ ],
4016
+ "layer_type": "full_attention",
4017
+ "is_auto": true,
4018
+ "dtype": "float16"
4019
+ },
4020
+ {
4021
+ "name": "past_key_values_17",
4022
+ "layer_index": 8,
4023
+ "shape": [
4024
+ 24,
4025
+ 8,
4026
+ 16384,
4027
+ 128
4028
+ ],
4029
+ "layer_type": "full_attention",
4030
+ "is_auto": true,
4031
+ "dtype": "float16"
4032
+ },
4033
+ {
4034
+ "name": "past_key_values_18",
4035
+ "layer_index": 9,
4036
+ "shape": [
4037
+ 24,
4038
+ 8,
4039
+ 16384,
4040
+ 128
4041
+ ],
4042
+ "layer_type": "full_attention",
4043
+ "is_auto": true,
4044
+ "dtype": "float16"
4045
+ },
4046
+ {
4047
+ "name": "past_key_values_19",
4048
+ "layer_index": 9,
4049
+ "shape": [
4050
+ 24,
4051
+ 8,
4052
+ 16384,
4053
+ 128
4054
+ ],
4055
+ "layer_type": "full_attention",
4056
+ "is_auto": true,
4057
+ "dtype": "float16"
4058
+ },
4059
+ {
4060
+ "name": "past_key_values_20",
4061
+ "layer_index": 10,
4062
+ "shape": [
4063
+ 24,
4064
+ 8,
4065
+ 16384,
4066
+ 128
4067
+ ],
4068
+ "layer_type": "full_attention",
4069
+ "is_auto": true,
4070
+ "dtype": "float16"
4071
+ },
4072
+ {
4073
+ "name": "past_key_values_21",
4074
+ "layer_index": 10,
4075
+ "shape": [
4076
+ 24,
4077
+ 8,
4078
+ 16384,
4079
+ 128
4080
+ ],
4081
+ "layer_type": "full_attention",
4082
+ "is_auto": true,
4083
+ "dtype": "float16"
4084
+ },
4085
+ {
4086
+ "name": "past_key_values_22",
4087
+ "layer_index": 11,
4088
+ "shape": [
4089
+ 24,
4090
+ 8,
4091
+ 16384,
4092
+ 128
4093
+ ],
4094
+ "layer_type": "full_attention",
4095
+ "is_auto": true,
4096
+ "dtype": "float16"
4097
+ },
4098
+ {
4099
+ "name": "past_key_values_23",
4100
+ "layer_index": 11,
4101
+ "shape": [
4102
+ 24,
4103
+ 8,
4104
+ 16384,
4105
+ 128
4106
+ ],
4107
+ "layer_type": "full_attention",
4108
+ "is_auto": true,
4109
+ "dtype": "float16"
4110
+ },
4111
+ {
4112
+ "name": "past_key_values_24",
4113
+ "layer_index": 12,
4114
+ "shape": [
4115
+ 24,
4116
+ 8,
4117
+ 16384,
4118
+ 128
4119
+ ],
4120
+ "layer_type": "full_attention",
4121
+ "is_auto": true,
4122
+ "dtype": "float16"
4123
+ },
4124
+ {
4125
+ "name": "past_key_values_25",
4126
+ "layer_index": 12,
4127
+ "shape": [
4128
+ 24,
4129
+ 8,
4130
+ 16384,
4131
+ 128
4132
+ ],
4133
+ "layer_type": "full_attention",
4134
+ "is_auto": true,
4135
+ "dtype": "float16"
4136
+ },
4137
+ {
4138
+ "name": "past_key_values_26",
4139
+ "layer_index": 13,
4140
+ "shape": [
4141
+ 24,
4142
+ 8,
4143
+ 16384,
4144
+ 128
4145
+ ],
4146
+ "layer_type": "full_attention",
4147
+ "is_auto": true,
4148
+ "dtype": "float16"
4149
+ },
4150
+ {
4151
+ "name": "past_key_values_27",
4152
+ "layer_index": 13,
4153
+ "shape": [
4154
+ 24,
4155
+ 8,
4156
+ 16384,
4157
+ 128
4158
+ ],
4159
+ "layer_type": "full_attention",
4160
+ "is_auto": true,
4161
+ "dtype": "float16"
4162
+ },
4163
+ {
4164
+ "name": "past_key_values_28",
4165
+ "layer_index": 14,
4166
+ "shape": [
4167
+ 24,
4168
+ 8,
4169
+ 16384,
4170
+ 128
4171
+ ],
4172
+ "layer_type": "full_attention",
4173
+ "is_auto": true,
4174
+ "dtype": "float16"
4175
+ },
4176
+ {
4177
+ "name": "past_key_values_29",
4178
+ "layer_index": 14,
4179
+ "shape": [
4180
+ 24,
4181
+ 8,
4182
+ 16384,
4183
+ 128
4184
+ ],
4185
+ "layer_type": "full_attention",
4186
+ "is_auto": true,
4187
+ "dtype": "float16"
4188
+ },
4189
+ {
4190
+ "name": "past_key_values_30",
4191
+ "layer_index": 15,
4192
+ "shape": [
4193
+ 24,
4194
+ 8,
4195
+ 16384,
4196
+ 128
4197
+ ],
4198
+ "layer_type": "full_attention",
4199
+ "is_auto": true,
4200
+ "dtype": "float16"
4201
+ },
4202
+ {
4203
+ "name": "past_key_values_31",
4204
+ "layer_index": 15,
4205
+ "shape": [
4206
+ 24,
4207
+ 8,
4208
+ 16384,
4209
+ 128
4210
+ ],
4211
+ "layer_type": "full_attention",
4212
+ "is_auto": true,
4213
+ "dtype": "float16"
4214
+ },
4215
+ {
4216
+ "name": "past_key_values_32",
4217
+ "layer_index": 16,
4218
+ "shape": [
4219
+ 24,
4220
+ 8,
4221
+ 16384,
4222
+ 128
4223
+ ],
4224
+ "layer_type": "full_attention",
4225
+ "is_auto": true,
4226
+ "dtype": "float16"
4227
+ },
4228
+ {
4229
+ "name": "past_key_values_33",
4230
+ "layer_index": 16,
4231
+ "shape": [
4232
+ 24,
4233
+ 8,
4234
+ 16384,
4235
+ 128
4236
+ ],
4237
+ "layer_type": "full_attention",
4238
+ "is_auto": true,
4239
+ "dtype": "float16"
4240
+ },
4241
+ {
4242
+ "name": "past_key_values_34",
4243
+ "layer_index": 17,
4244
+ "shape": [
4245
+ 24,
4246
+ 8,
4247
+ 16384,
4248
+ 128
4249
+ ],
4250
+ "layer_type": "full_attention",
4251
+ "is_auto": true,
4252
+ "dtype": "float16"
4253
+ },
4254
+ {
4255
+ "name": "past_key_values_35",
4256
+ "layer_index": 17,
4257
+ "shape": [
4258
+ 24,
4259
+ 8,
4260
+ 16384,
4261
+ 128
4262
+ ],
4263
+ "layer_type": "full_attention",
4264
+ "is_auto": true,
4265
+ "dtype": "float16"
4266
+ },
4267
+ {
4268
+ "name": "past_key_values_36",
4269
+ "layer_index": 18,
4270
+ "shape": [
4271
+ 24,
4272
+ 8,
4273
+ 16384,
4274
+ 128
4275
+ ],
4276
+ "layer_type": "full_attention",
4277
+ "is_auto": true,
4278
+ "dtype": "float16"
4279
+ },
4280
+ {
4281
+ "name": "past_key_values_37",
4282
+ "layer_index": 18,
4283
+ "shape": [
4284
+ 24,
4285
+ 8,
4286
+ 16384,
4287
+ 128
4288
+ ],
4289
+ "layer_type": "full_attention",
4290
+ "is_auto": true,
4291
+ "dtype": "float16"
4292
+ },
4293
+ {
4294
+ "name": "past_key_values_38",
4295
+ "layer_index": 19,
4296
+ "shape": [
4297
+ 24,
4298
+ 8,
4299
+ 16384,
4300
+ 128
4301
+ ],
4302
+ "layer_type": "full_attention",
4303
+ "is_auto": true,
4304
+ "dtype": "float16"
4305
+ },
4306
+ {
4307
+ "name": "past_key_values_39",
4308
+ "layer_index": 19,
4309
+ "shape": [
4310
+ 24,
4311
+ 8,
4312
+ 16384,
4313
+ 128
4314
+ ],
4315
+ "layer_type": "full_attention",
4316
+ "is_auto": true,
4317
+ "dtype": "float16"
4318
+ },
4319
+ {
4320
+ "name": "past_key_values_40",
4321
+ "layer_index": 20,
4322
+ "shape": [
4323
+ 24,
4324
+ 8,
4325
+ 16384,
4326
+ 128
4327
+ ],
4328
+ "layer_type": "full_attention",
4329
+ "is_auto": true,
4330
+ "dtype": "float16"
4331
+ },
4332
+ {
4333
+ "name": "past_key_values_41",
4334
+ "layer_index": 20,
4335
+ "shape": [
4336
+ 24,
4337
+ 8,
4338
+ 16384,
4339
+ 128
4340
+ ],
4341
+ "layer_type": "full_attention",
4342
+ "is_auto": true,
4343
+ "dtype": "float16"
4344
+ },
4345
+ {
4346
+ "name": "past_key_values_42",
4347
+ "layer_index": 21,
4348
+ "shape": [
4349
+ 24,
4350
+ 8,
4351
+ 16384,
4352
+ 128
4353
+ ],
4354
+ "layer_type": "full_attention",
4355
+ "is_auto": true,
4356
+ "dtype": "float16"
4357
+ },
4358
+ {
4359
+ "name": "past_key_values_43",
4360
+ "layer_index": 21,
4361
+ "shape": [
4362
+ 24,
4363
+ 8,
4364
+ 16384,
4365
+ 128
4366
+ ],
4367
+ "layer_type": "full_attention",
4368
+ "is_auto": true,
4369
+ "dtype": "float16"
4370
+ },
4371
+ {
4372
+ "name": "past_key_values_44",
4373
+ "layer_index": 22,
4374
+ "shape": [
4375
+ 24,
4376
+ 8,
4377
+ 16384,
4378
+ 128
4379
+ ],
4380
+ "layer_type": "full_attention",
4381
+ "is_auto": true,
4382
+ "dtype": "float16"
4383
+ },
4384
+ {
4385
+ "name": "past_key_values_45",
4386
+ "layer_index": 22,
4387
+ "shape": [
4388
+ 24,
4389
+ 8,
4390
+ 16384,
4391
+ 128
4392
+ ],
4393
+ "layer_type": "full_attention",
4394
+ "is_auto": true,
4395
+ "dtype": "float16"
4396
+ },
4397
+ {
4398
+ "name": "past_key_values_46",
4399
+ "layer_index": 23,
4400
+ "shape": [
4401
+ 24,
4402
+ 8,
4403
+ 16384,
4404
+ 128
4405
+ ],
4406
+ "layer_type": "full_attention",
4407
+ "is_auto": true,
4408
+ "dtype": "float16"
4409
+ },
4410
+ {
4411
+ "name": "past_key_values_47",
4412
+ "layer_index": 23,
4413
+ "shape": [
4414
+ 24,
4415
+ 8,
4416
+ 16384,
4417
+ 128
4418
+ ],
4419
+ "layer_type": "full_attention",
4420
+ "is_auto": true,
4421
+ "dtype": "float16"
4422
+ },
4423
+ {
4424
+ "name": "past_key_values_48",
4425
+ "layer_index": 24,
4426
+ "shape": [
4427
+ 24,
4428
+ 8,
4429
+ 16384,
4430
+ 128
4431
+ ],
4432
+ "layer_type": "full_attention",
4433
+ "is_auto": true,
4434
+ "dtype": "float16"
4435
+ },
4436
+ {
4437
+ "name": "past_key_values_49",
4438
+ "layer_index": 24,
4439
+ "shape": [
4440
+ 24,
4441
+ 8,
4442
+ 16384,
4443
+ 128
4444
+ ],
4445
+ "layer_type": "full_attention",
4446
+ "is_auto": true,
4447
+ "dtype": "float16"
4448
+ },
4449
+ {
4450
+ "name": "past_key_values_50",
4451
+ "layer_index": 25,
4452
+ "shape": [
4453
+ 24,
4454
+ 8,
4455
+ 16384,
4456
+ 128
4457
+ ],
4458
+ "layer_type": "full_attention",
4459
+ "is_auto": true,
4460
+ "dtype": "float16"
4461
+ },
4462
+ {
4463
+ "name": "past_key_values_51",
4464
+ "layer_index": 25,
4465
+ "shape": [
4466
+ 24,
4467
+ 8,
4468
+ 16384,
4469
+ 128
4470
+ ],
4471
+ "layer_type": "full_attention",
4472
+ "is_auto": true,
4473
+ "dtype": "float16"
4474
+ },
4475
+ {
4476
+ "name": "past_key_values_52",
4477
+ "layer_index": 26,
4478
+ "shape": [
4479
+ 24,
4480
+ 8,
4481
+ 16384,
4482
+ 128
4483
+ ],
4484
+ "layer_type": "full_attention",
4485
+ "is_auto": true,
4486
+ "dtype": "float16"
4487
+ },
4488
+ {
4489
+ "name": "past_key_values_53",
4490
+ "layer_index": 26,
4491
+ "shape": [
4492
+ 24,
4493
+ 8,
4494
+ 16384,
4495
+ 128
4496
+ ],
4497
+ "layer_type": "full_attention",
4498
+ "is_auto": true,
4499
+ "dtype": "float16"
4500
+ },
4501
+ {
4502
+ "name": "past_key_values_54",
4503
+ "layer_index": 27,
4504
+ "shape": [
4505
+ 24,
4506
+ 8,
4507
+ 16384,
4508
+ 128
4509
+ ],
4510
+ "layer_type": "full_attention",
4511
+ "is_auto": true,
4512
+ "dtype": "float16"
4513
+ },
4514
+ {
4515
+ "name": "past_key_values_55",
4516
+ "layer_index": 27,
4517
+ "shape": [
4518
+ 24,
4519
+ 8,
4520
+ 16384,
4521
+ 128
4522
+ ],
4523
+ "layer_type": "full_attention",
4524
+ "is_auto": true,
4525
+ "dtype": "float16"
4526
+ },
4527
+ {
4528
+ "name": "past_key_values_56",
4529
+ "layer_index": 28,
4530
+ "shape": [
4531
+ 24,
4532
+ 8,
4533
+ 16384,
4534
+ 128
4535
+ ],
4536
+ "layer_type": "full_attention",
4537
+ "is_auto": true,
4538
+ "dtype": "float16"
4539
+ },
4540
+ {
4541
+ "name": "past_key_values_57",
4542
+ "layer_index": 28,
4543
+ "shape": [
4544
+ 24,
4545
+ 8,
4546
+ 16384,
4547
+ 128
4548
+ ],
4549
+ "layer_type": "full_attention",
4550
+ "is_auto": true,
4551
+ "dtype": "float16"
4552
+ },
4553
+ {
4554
+ "name": "past_key_values_58",
4555
+ "layer_index": 29,
4556
+ "shape": [
4557
+ 24,
4558
+ 8,
4559
+ 16384,
4560
+ 128
4561
+ ],
4562
+ "layer_type": "full_attention",
4563
+ "is_auto": true,
4564
+ "dtype": "float16"
4565
+ },
4566
+ {
4567
+ "name": "past_key_values_59",
4568
+ "layer_index": 29,
4569
+ "shape": [
4570
+ 24,
4571
+ 8,
4572
+ 16384,
4573
+ 128
4574
+ ],
4575
+ "layer_type": "full_attention",
4576
+ "is_auto": true,
4577
+ "dtype": "float16"
4578
+ },
4579
+ {
4580
+ "name": "past_key_values_60",
4581
+ "layer_index": 30,
4582
+ "shape": [
4583
+ 24,
4584
+ 8,
4585
+ 16384,
4586
+ 128
4587
+ ],
4588
+ "layer_type": "full_attention",
4589
+ "is_auto": true,
4590
+ "dtype": "float16"
4591
+ },
4592
+ {
4593
+ "name": "past_key_values_61",
4594
+ "layer_index": 30,
4595
+ "shape": [
4596
+ 24,
4597
+ 8,
4598
+ 16384,
4599
+ 128
4600
+ ],
4601
+ "layer_type": "full_attention",
4602
+ "is_auto": true,
4603
+ "dtype": "float16"
4604
+ },
4605
+ {
4606
+ "name": "past_key_values_62",
4607
+ "layer_index": 31,
4608
+ "shape": [
4609
+ 24,
4610
+ 8,
4611
+ 16384,
4612
+ 128
4613
+ ],
4614
+ "layer_type": "full_attention",
4615
+ "is_auto": true,
4616
+ "dtype": "float16"
4617
+ },
4618
+ {
4619
+ "name": "past_key_values_63",
4620
+ "layer_index": 31,
4621
+ "shape": [
4622
+ 24,
4623
+ 8,
4624
+ 16384,
4625
+ 128
4626
+ ],
4627
+ "layer_type": "full_attention",
4628
+ "is_auto": true,
4629
+ "dtype": "float16"
4630
+ },
4631
+ {
4632
+ "name": "past_key_values_64",
4633
+ "layer_index": 32,
4634
+ "shape": [
4635
+ 24,
4636
+ 8,
4637
+ 16384,
4638
+ 128
4639
+ ],
4640
+ "layer_type": "full_attention",
4641
+ "is_auto": true,
4642
+ "dtype": "float16"
4643
+ },
4644
+ {
4645
+ "name": "past_key_values_65",
4646
+ "layer_index": 32,
4647
+ "shape": [
4648
+ 24,
4649
+ 8,
4650
+ 16384,
4651
+ 128
4652
+ ],
4653
+ "layer_type": "full_attention",
4654
+ "is_auto": true,
4655
+ "dtype": "float16"
4656
+ },
4657
+ {
4658
+ "name": "past_key_values_66",
4659
+ "layer_index": 33,
4660
+ "shape": [
4661
+ 24,
4662
+ 8,
4663
+ 16384,
4664
+ 128
4665
+ ],
4666
+ "layer_type": "full_attention",
4667
+ "is_auto": true,
4668
+ "dtype": "float16"
4669
+ },
4670
+ {
4671
+ "name": "past_key_values_67",
4672
+ "layer_index": 33,
4673
+ "shape": [
4674
+ 24,
4675
+ 8,
4676
+ 16384,
4677
+ 128
4678
+ ],
4679
+ "layer_type": "full_attention",
4680
+ "is_auto": true,
4681
+ "dtype": "float16"
4682
+ },
4683
+ {
4684
+ "name": "past_key_values_68",
4685
+ "layer_index": 34,
4686
+ "shape": [
4687
+ 24,
4688
+ 8,
4689
+ 16384,
4690
+ 128
4691
+ ],
4692
+ "layer_type": "full_attention",
4693
+ "is_auto": true,
4694
+ "dtype": "float16"
4695
+ },
4696
+ {
4697
+ "name": "past_key_values_69",
4698
+ "layer_index": 34,
4699
+ "shape": [
4700
+ 24,
4701
+ 8,
4702
+ 16384,
4703
+ 128
4704
+ ],
4705
+ "layer_type": "full_attention",
4706
+ "is_auto": true,
4707
+ "dtype": "float16"
4708
+ },
4709
+ {
4710
+ "name": "past_key_values_70",
4711
+ "layer_index": 35,
4712
+ "shape": [
4713
+ 24,
4714
+ 8,
4715
+ 16384,
4716
+ 128
4717
+ ],
4718
+ "layer_type": "full_attention",
4719
+ "is_auto": true,
4720
+ "dtype": "float16"
4721
+ },
4722
+ {
4723
+ "name": "past_key_values_71",
4724
+ "layer_index": 35,
4725
+ "shape": [
4726
+ 24,
4727
+ 8,
4728
+ 16384,
4729
+ 128
4730
+ ],
4731
+ "layer_type": "full_attention",
4732
+ "is_auto": true,
4733
+ "dtype": "float16"
4734
+ }
4735
+ ]
4736
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0acdaba32b920d640afb36af4396c91974e074735636e4016d17a8ed9c03730
3
+ size 11422753
tokenizer_config.json ADDED
@@ -0,0 +1,243 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ },
181
+ "151665": {
182
+ "content": "<tool_response>",
183
+ "lstrip": false,
184
+ "normalized": false,
185
+ "rstrip": false,
186
+ "single_word": false,
187
+ "special": false
188
+ },
189
+ "151666": {
190
+ "content": "</tool_response>",
191
+ "lstrip": false,
192
+ "normalized": false,
193
+ "rstrip": false,
194
+ "single_word": false,
195
+ "special": false
196
+ },
197
+ "151667": {
198
+ "content": "<think>",
199
+ "lstrip": false,
200
+ "normalized": false,
201
+ "rstrip": false,
202
+ "single_word": false,
203
+ "special": false
204
+ },
205
+ "151668": {
206
+ "content": "</think>",
207
+ "lstrip": false,
208
+ "normalized": false,
209
+ "rstrip": false,
210
+ "single_word": false,
211
+ "special": false
212
+ }
213
+ },
214
+ "additional_special_tokens": [
215
+ "<|im_start|>",
216
+ "<|im_end|>",
217
+ "<|object_ref_start|>",
218
+ "<|object_ref_end|>",
219
+ "<|box_start|>",
220
+ "<|box_end|>",
221
+ "<|quad_start|>",
222
+ "<|quad_end|>",
223
+ "<|vision_start|>",
224
+ "<|vision_end|>",
225
+ "<|vision_pad|>",
226
+ "<|image_pad|>",
227
+ "<|video_pad|>"
228
+ ],
229
+ "bos_token": null,
230
+ "clean_up_tokenization_spaces": false,
231
+ "eos_token": "<|im_end|>",
232
+ "errors": "replace",
233
+ "extra_special_tokens": {},
234
+ "max_length": 2048,
235
+ "model_max_length": 131072,
236
+ "pad_token": "<|endoftext|>",
237
+ "split_special_tokens": false,
238
+ "stride": 0,
239
+ "tokenizer_class": "Qwen2Tokenizer",
240
+ "truncation_side": "right",
241
+ "truncation_strategy": "longest_first",
242
+ "unk_token": null
243
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff