Hugging Face's logo

MBZUAI
/
MedMO-4B

Image-Text-to-Text
Transformers
Safetensors
qwen3_vl
medical
multimodal
grounding
report-generation
radiology
clinical-reasoning
mri
ct
histopathology
x-ray
fundus
conversational
Model card Files
xet
 Community
2
ankanmbz commited on
Commit
6897cad
·
verified ·
1 Parent(s): ad782c0

Delete checkpoint-2113

Browse files
Files changed (19) hide show
  1. checkpoint-2113/added_tokens.json +0 -28
  2. checkpoint-2113/chat_template.jinja +0 -120
  3. checkpoint-2113/config.json +0 -67
  4. checkpoint-2113/generation_config.json +0 -13
  5. checkpoint-2113/latest +0 -1
  6. checkpoint-2113/merges.txt +0 -0
  7. checkpoint-2113/model-00001-of-00002.safetensors +0 -3
  8. checkpoint-2113/model-00002-of-00002.safetensors +0 -3
  9. checkpoint-2113/model.safetensors.index.json +0 -722
  10. checkpoint-2113/preprocessor_config.json +0 -39
  11. checkpoint-2113/scheduler.pt +0 -3
  12. checkpoint-2113/special_tokens_map.json +0 -31
  13. checkpoint-2113/tokenizer.json +0 -3
  14. checkpoint-2113/tokenizer_config.json +0 -242
  15. checkpoint-2113/trainer_state.json +0 -2144
  16. checkpoint-2113/training_args.bin +0 -3
  17. checkpoint-2113/video_preprocessor_config.json +0 -41
  18. checkpoint-2113/vocab.json +0 -0
  19. checkpoint-2113/zero_to_fp32.py +0 -760
checkpoint-2113/added_tokens.json DELETED
@@ -1,28 +0,0 @@
1
- {
2
- "</think>": 151668,
3
- "</tool_call>": 151658,
4
- "</tool_response>": 151666,
5
- "<think>": 151667,
6
- "<tool_call>": 151657,
7
- "<tool_response>": 151665,
8
- "<|box_end|>": 151649,
9
- "<|box_start|>": 151648,
10
- "<|endoftext|>": 151643,
11
- "<|file_sep|>": 151664,
12
- "<|fim_middle|>": 151660,
13
- "<|fim_pad|>": 151662,
14
- "<|fim_prefix|>": 151659,
15
- "<|fim_suffix|>": 151661,
16
- "<|im_end|>": 151645,
17
- "<|im_start|>": 151644,
18
- "<|image_pad|>": 151655,
19
- "<|object_ref_end|>": 151647,
20
- "<|object_ref_start|>": 151646,
21
- "<|quad_end|>": 151651,
22
- "<|quad_start|>": 151650,
23
- "<|repo_name|>": 151663,
24
- "<|video_pad|>": 151656,
25
- "<|vision_end|>": 151653,
26
- "<|vision_pad|>": 151654,
27
- "<|vision_start|>": 151652
28
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/chat_template.jinja DELETED
@@ -1,120 +0,0 @@
1
- {%- if tools %}
2
- {{- '<|im_start|>system\n' }}
3
- {%- if messages[0].role == 'system' %}
4
- {%- if messages[0].content is string %}
5
- {{- messages[0].content }}
6
- {%- else %}
7
- {%- for content in messages[0].content %}
8
- {%- if 'text' in content %}
9
- {{- content.text }}
10
- {%- endif %}
11
- {%- endfor %}
12
- {%- endif %}
13
- {{- '\n\n' }}
14
- {%- endif %}
15
- {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
16
- {%- for tool in tools %}
17
- {{- "\n" }}
18
- {{- tool | tojson }}
19
- {%- endfor %}
20
- {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
21
- {%- else %}
22
- {%- if messages[0].role == 'system' %}
23
- {{- '<|im_start|>system\n' }}
24
- {%- if messages[0].content is string %}
25
- {{- messages[0].content }}
26
- {%- else %}
27
- {%- for content in messages[0].content %}
28
- {%- if 'text' in content %}
29
- {{- content.text }}
30
- {%- endif %}
31
- {%- endfor %}
32
- {%- endif %}
33
- {{- '<|im_end|>\n' }}
34
- {%- endif %}
35
- {%- endif %}
36
- {%- set image_count = namespace(value=0) %}
37
- {%- set video_count = namespace(value=0) %}
38
- {%- for message in messages %}
39
- {%- if message.role == "user" %}
40
- {{- '<|im_start|>' + message.role + '\n' }}
41
- {%- if message.content is string %}
42
- {{- message.content }}
43
- {%- else %}
44
- {%- for content in message.content %}
45
- {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
46
- {%- set image_count.value = image_count.value + 1 %}
47
- {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
48
- <|vision_start|><|image_pad|><|vision_end|>
49
- {%- elif content.type == 'video' or 'video' in content %}
50
- {%- set video_count.value = video_count.value + 1 %}
51
- {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
52
- <|vision_start|><|video_pad|><|vision_end|>
53
- {%- elif 'text' in content %}
54
- {{- content.text }}
55
- {%- endif %}
56
- {%- endfor %}
57
- {%- endif %}
58
- {{- '<|im_end|>\n' }}
59
- {%- elif message.role == "assistant" %}
60
- {{- '<|im_start|>' + message.role + '\n' }}
61
- {%- if message.content is string %}
62
- {{- message.content }}
63
- {%- else %}
64
- {%- for content_item in message.content %}
65
- {%- if 'text' in content_item %}
66
- {{- content_item.text }}
67
- {%- endif %}
68
- {%- endfor %}
69
- {%- endif %}
70
- {%- if message.tool_calls %}
71
- {%- for tool_call in message.tool_calls %}
72
- {%- if (loop.first and message.content) or (not loop.first) %}
73
- {{- '\n' }}
74
- {%- endif %}
75
- {%- if tool_call.function %}
76
- {%- set tool_call = tool_call.function %}
77
- {%- endif %}
78
- {{- '<tool_call>\n{"name": "' }}
79
- {{- tool_call.name }}
80
- {{- '", "arguments": ' }}
81
- {%- if tool_call.arguments is string %}
82
- {{- tool_call.arguments }}
83
- {%- else %}
84
- {{- tool_call.arguments | tojson }}
85
- {%- endif %}
86
- {{- '}\n</tool_call>' }}
87
- {%- endfor %}
88
- {%- endif %}
89
- {{- '<|im_end|>\n' }}
90
- {%- elif message.role == "tool" %}
91
- {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
92
- {{- '<|im_start|>user' }}
93
- {%- endif %}
94
- {{- '\n<tool_response>\n' }}
95
- {%- if message.content is string %}
96
- {{- message.content }}
97
- {%- else %}
98
- {%- for content in message.content %}
99
- {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
100
- {%- set image_count.value = image_count.value + 1 %}
101
- {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
102
- <|vision_start|><|image_pad|><|vision_end|>
103
- {%- elif content.type == 'video' or 'video' in content %}
104
- {%- set video_count.value = video_count.value + 1 %}
105
- {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
106
- <|vision_start|><|video_pad|><|vision_end|>
107
- {%- elif 'text' in content %}
108
- {{- content.text }}
109
- {%- endif %}
110
- {%- endfor %}
111
- {%- endif %}
112
- {{- '\n</tool_response>' }}
113
- {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
114
- {{- '<|im_end|>\n' }}
115
- {%- endif %}
116
- {%- endif %}
117
- {%- endfor %}
118
- {%- if add_generation_prompt %}
119
- {{- '<|im_start|>assistant\n' }}
120
- {%- endif %}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/config.json DELETED
@@ -1,67 +0,0 @@
1
- {
2
- "architectures": [
3
- "Qwen3VLForConditionalGeneration"
4
- ],
5
- "dtype": "bfloat16",
6
- "eos_token_id": 151645,
7
- "image_token_id": 151655,
8
- "model_type": "qwen3_vl",
9
- "pad_token_id": 151643,
10
- "text_config": {
11
- "attention_bias": false,
12
- "attention_dropout": 0.0,
13
- "bos_token_id": 151643,
14
- "dtype": "bfloat16",
15
- "eos_token_id": 151645,
16
- "head_dim": 128,
17
- "hidden_act": "silu",
18
- "hidden_size": 2560,
19
- "initializer_range": 0.02,
20
- "intermediate_size": 9728,
21
- "max_position_embeddings": 262144,
22
- "model_type": "qwen3_vl_text",
23
- "num_attention_heads": 32,
24
- "num_hidden_layers": 36,
25
- "num_key_value_heads": 8,
26
- "rms_norm_eps": 1e-06,
27
- "rope_scaling": {
28
- "mrope_interleaved": true,
29
- "mrope_section": [
30
- 24,
31
- 20,
32
- 20
33
- ],
34
- "rope_type": "default"
35
- },
36
- "rope_theta": 5000000,
37
- "tie_word_embeddings": true,
38
- "use_cache": true,
39
- "vocab_size": 151936
40
- },
41
- "tie_word_embeddings": true,
42
- "transformers_version": "4.57.1",
43
- "video_token_id": 151656,
44
- "vision_config": {
45
- "deepstack_visual_indexes": [
46
- 5,
47
- 11,
48
- 17
49
- ],
50
- "depth": 24,
51
- "dtype": "bfloat16",
52
- "hidden_act": "gelu_pytorch_tanh",
53
- "hidden_size": 1024,
54
- "in_channels": 3,
55
- "initializer_range": 0.02,
56
- "intermediate_size": 4096,
57
- "model_type": "qwen3_vl",
58
- "num_heads": 16,
59
- "num_position_embeddings": 2304,
60
- "out_hidden_size": 2560,
61
- "patch_size": 16,
62
- "spatial_merge_size": 2,
63
- "temporal_patch_size": 2
64
- },
65
- "vision_end_token_id": 151653,
66
- "vision_start_token_id": 151652
67
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/generation_config.json DELETED
@@ -1,13 +0,0 @@
1
- {
2
- "do_sample": true,
3
- "eos_token_id": [
4
- 151645,
5
- 151645,
6
- 151643
7
- ],
8
- "pad_token_id": 151643,
9
- "temperature": 0.7,
10
- "top_k": 20,
11
- "top_p": 0.8,
12
- "transformers_version": "4.57.1"
13
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/latest DELETED
@@ -1 +0,0 @@
1
- global_step2113
 
 
checkpoint-2113/merges.txt DELETED
The diff for this file is too large to render. See raw diff
 
checkpoint-2113/model-00001-of-00002.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6fd00c1db433ecfc23f2b39abde45f0210109a55f45092a8c169f61fb37114f9
3
- size 4990497880
 
 
 
 
checkpoint-2113/model-00002-of-00002.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8e55746c433dfe11214a27d007edb46aaefad5782a13ca2fee8ca7fa559aebbc
3
- size 4663133960
 
 
 
 
checkpoint-2113/model.safetensors.index.json DELETED
@@ -1,722 +0,0 @@
1
- {
2
- "metadata": {
3
- "total_parameters": 4437815808,
4
- "total_size": 9653543936
5
- },
6
- "weight_map": {
7
- "lm_head.weight": "model-00002-of-00002.safetensors",
8
- "model.language_model.embed_tokens.weight": "model-00001-of-00002.safetensors",
9
- "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
10
- "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
11
- "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
12
- "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
13
- "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
14
- "model.language_model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
15
- "model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
16
- "model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
17
- "model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
18
- "model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
19
- "model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
20
- "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
21
- "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
22
- "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
23
- "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
24
- "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
25
- "model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
26
- "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
27
- "model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
28
- "model.language_model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
29
- "model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
30
- "model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
31
- "model.language_model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
32
- "model.language_model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
33
- "model.language_model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
34
- "model.language_model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
35
- "model.language_model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
36
- "model.language_model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
37
- "model.language_model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
38
- "model.language_model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
39
- "model.language_model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
40
- "model.language_model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
41
- "model.language_model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
42
- "model.language_model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
43
- "model.language_model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
44
- "model.language_model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
45
- "model.language_model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
46
- "model.language_model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
47
- "model.language_model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
48
- "model.language_model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
49
- "model.language_model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
50
- "model.language_model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
51
- "model.language_model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
52
- "model.language_model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
53
- "model.language_model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
54
- "model.language_model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
55
- "model.language_model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
56
- "model.language_model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
57
- "model.language_model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
58
- "model.language_model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
59
- "model.language_model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
60
- "model.language_model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
61
- "model.language_model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
62
- "model.language_model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
63
- "model.language_model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
64
- "model.language_model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
65
- "model.language_model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
66
- "model.language_model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
67
- "model.language_model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
68
- "model.language_model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
69
- "model.language_model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
70
- "model.language_model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
71
- "model.language_model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
72
- "model.language_model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
73
- "model.language_model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
74
- "model.language_model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
75
- "model.language_model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
76
- "model.language_model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
77
- "model.language_model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
78
- "model.language_model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
79
- "model.language_model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
80
- "model.language_model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
81
- "model.language_model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
82
- "model.language_model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
83
- "model.language_model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
84
- "model.language_model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
85
- "model.language_model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
86
- "model.language_model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
87
- "model.language_model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
88
- "model.language_model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
89
- "model.language_model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
90
- "model.language_model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
91
- "model.language_model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
92
- "model.language_model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
93
- "model.language_model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
94
- "model.language_model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
95
- "model.language_model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
96
- "model.language_model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
97
- "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors",
98
- "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
99
- "model.language_model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
100
- "model.language_model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
101
- "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
102
- "model.language_model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
103
- "model.language_model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
104
- "model.language_model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
105
- "model.language_model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
106
- "model.language_model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
107
- "model.language_model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
108
- "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
109
- "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
110
- "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
111
- "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
112
- "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
113
- "model.language_model.layers.17.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
114
- "model.language_model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
115
- "model.language_model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
116
- "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
117
- "model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
118
- "model.language_model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
119
- "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors",
120
- "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
121
- "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
122
- "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
123
- "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
124
- "model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
125
- "model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
126
- "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
127
- "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
128
- "model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
129
- "model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
130
- "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
131
- "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
132
- "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
133
- "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
134
- "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
135
- "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
136
- "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
137
- "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
138
- "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
139
- "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
140
- "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
141
- "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
142
- "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
143
- "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
144
- "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
145
- "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
146
- "model.language_model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
147
- "model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
148
- "model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
149
- "model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
150
- "model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
151
- "model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
152
- "model.language_model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
153
- "model.language_model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
154
- "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
155
- "model.language_model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
156
- "model.language_model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
157
- "model.language_model.layers.20.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
158
- "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
159
- "model.language_model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
160
- "model.language_model.layers.20.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
161
- "model.language_model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
162
- "model.language_model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
163
- "model.language_model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
164
- "model.language_model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
165
- "model.language_model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
166
- "model.language_model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
167
- "model.language_model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
168
- "model.language_model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
169
- "model.language_model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
170
- "model.language_model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
171
- "model.language_model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
172
- "model.language_model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
173
- "model.language_model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
174
- "model.language_model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
175
- "model.language_model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
176
- "model.language_model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
177
- "model.language_model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
178
- "model.language_model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
179
- "model.language_model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
180
- "model.language_model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
181
- "model.language_model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
182
- "model.language_model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
183
- "model.language_model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
184
- "model.language_model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
185
- "model.language_model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
186
- "model.language_model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
187
- "model.language_model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
188
- "model.language_model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
189
- "model.language_model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
190
- "model.language_model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
191
- "model.language_model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
192
- "model.language_model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
193
- "model.language_model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
194
- "model.language_model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
195
- "model.language_model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
196
- "model.language_model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
197
- "model.language_model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
198
- "model.language_model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
199
- "model.language_model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
200
- "model.language_model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
201
- "model.language_model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
202
- "model.language_model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
203
- "model.language_model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
204
- "model.language_model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
205
- "model.language_model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
206
- "model.language_model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
207
- "model.language_model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
208
- "model.language_model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
209
- "model.language_model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
210
- "model.language_model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
211
- "model.language_model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
212
- "model.language_model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
213
- "model.language_model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
214
- "model.language_model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
215
- "model.language_model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
216
- "model.language_model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
217
- "model.language_model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
218
- "model.language_model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
219
- "model.language_model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
220
- "model.language_model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
221
- "model.language_model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
222
- "model.language_model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
223
- "model.language_model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
224
- "model.language_model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
225
- "model.language_model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
226
- "model.language_model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
227
- "model.language_model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
228
- "model.language_model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
229
- "model.language_model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
230
- "model.language_model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
231
- "model.language_model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
232
- "model.language_model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
233
- "model.language_model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
234
- "model.language_model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
235
- "model.language_model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
236
- "model.language_model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
237
- "model.language_model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
238
- "model.language_model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
239
- "model.language_model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
240
- "model.language_model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
241
- "model.language_model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
242
- "model.language_model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
243
- "model.language_model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
244
- "model.language_model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
245
- "model.language_model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
246
- "model.language_model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
247
- "model.language_model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
248
- "model.language_model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
249
- "model.language_model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
250
- "model.language_model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
251
- "model.language_model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
252
- "model.language_model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
253
- "model.language_model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
254
- "model.language_model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
255
- "model.language_model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
256
- "model.language_model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
257
- "model.language_model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
258
- "model.language_model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
259
- "model.language_model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
260
- "model.language_model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
261
- "model.language_model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
262
- "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
263
- "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
264
- "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
265
- "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
266
- "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
267
- "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
268
- "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
269
- "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
270
- "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
271
- "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
272
- "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
273
- "model.language_model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
274
- "model.language_model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
275
- "model.language_model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
276
- "model.language_model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
277
- "model.language_model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
278
- "model.language_model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
279
- "model.language_model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
280
- "model.language_model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
281
- "model.language_model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
282
- "model.language_model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
283
- "model.language_model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
284
- "model.language_model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
285
- "model.language_model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
286
- "model.language_model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
287
- "model.language_model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
288
- "model.language_model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
289
- "model.language_model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
290
- "model.language_model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
291
- "model.language_model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
292
- "model.language_model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
293
- "model.language_model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
294
- "model.language_model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
295
- "model.language_model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
296
- "model.language_model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
297
- "model.language_model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
298
- "model.language_model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
299
- "model.language_model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
300
- "model.language_model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
301
- "model.language_model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
302
- "model.language_model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
303
- "model.language_model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
304
- "model.language_model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
305
- "model.language_model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
306
- "model.language_model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
307
- "model.language_model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
308
- "model.language_model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
309
- "model.language_model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
310
- "model.language_model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
311
- "model.language_model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
312
- "model.language_model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
313
- "model.language_model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
314
- "model.language_model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
315
- "model.language_model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
316
- "model.language_model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
317
- "model.language_model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
318
- "model.language_model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
319
- "model.language_model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
320
- "model.language_model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
321
- "model.language_model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
322
- "model.language_model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
323
- "model.language_model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
324
- "model.language_model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
325
- "model.language_model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
326
- "model.language_model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
327
- "model.language_model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
328
- "model.language_model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
329
- "model.language_model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
330
- "model.language_model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
331
- "model.language_model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
332
- "model.language_model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
333
- "model.language_model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
334
- "model.language_model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
335
- "model.language_model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
336
- "model.language_model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
337
- "model.language_model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
338
- "model.language_model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
339
- "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
340
- "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
341
- "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
342
- "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
343
- "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
344
- "model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
345
- "model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
346
- "model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
347
- "model.language_model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
348
- "model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
349
- "model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
350
- "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
351
- "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
352
- "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
353
- "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
354
- "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
355
- "model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
356
- "model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
357
- "model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
358
- "model.language_model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
359
- "model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
360
- "model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
361
- "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
362
- "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
363
- "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
364
- "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
365
- "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
366
- "model.language_model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
367
- "model.language_model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
368
- "model.language_model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
369
- "model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
370
- "model.language_model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
371
- "model.language_model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
372
- "model.language_model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
373
- "model.language_model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
374
- "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
375
- "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
376
- "model.language_model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
377
- "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
378
- "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
379
- "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
380
- "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
381
- "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
382
- "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
383
- "model.language_model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
384
- "model.language_model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
385
- "model.language_model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
386
- "model.language_model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
387
- "model.language_model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
388
- "model.language_model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
389
- "model.language_model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
390
- "model.language_model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
391
- "model.language_model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
392
- "model.language_model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
393
- "model.language_model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
394
- "model.language_model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
395
- "model.language_model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
396
- "model.language_model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
397
- "model.language_model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
398
- "model.language_model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
399
- "model.language_model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
400
- "model.language_model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
401
- "model.language_model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
402
- "model.language_model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
403
- "model.language_model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
404
- "model.language_model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
405
- "model.language_model.norm.weight": "model-00002-of-00002.safetensors",
406
- "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors",
407
- "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors",
408
- "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors",
409
- "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors",
410
- "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
411
- "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
412
- "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
413
- "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
414
- "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors",
415
- "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors",
416
- "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors",
417
- "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors",
418
- "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors",
419
- "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors",
420
- "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors",
421
- "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors",
422
- "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
423
- "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
424
- "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
425
- "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
426
- "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors",
427
- "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors",
428
- "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors",
429
- "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors",
430
- "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors",
431
- "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors",
432
- "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors",
433
- "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors",
434
- "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
435
- "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
436
- "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
437
- "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
438
- "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors",
439
- "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors",
440
- "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors",
441
- "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors",
442
- "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors",
443
- "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors",
444
- "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors",
445
- "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors",
446
- "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
447
- "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
448
- "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
449
- "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
450
- "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors",
451
- "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors",
452
- "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors",
453
- "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors",
454
- "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors",
455
- "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors",
456
- "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors",
457
- "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors",
458
- "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
459
- "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
460
- "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
461
- "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
462
- "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors",
463
- "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors",
464
- "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors",
465
- "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors",
466
- "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors",
467
- "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors",
468
- "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors",
469
- "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors",
470
- "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
471
- "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
472
- "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
473
- "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
474
- "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors",
475
- "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors",
476
- "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors",
477
- "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors",
478
- "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors",
479
- "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors",
480
- "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors",
481
- "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors",
482
- "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
483
- "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
484
- "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
485
- "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
486
- "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors",
487
- "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors",
488
- "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors",
489
- "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors",
490
- "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors",
491
- "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors",
492
- "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors",
493
- "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors",
494
- "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
495
- "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
496
- "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
497
- "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
498
- "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors",
499
- "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors",
500
- "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors",
501
- "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors",
502
- "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors",
503
- "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors",
504
- "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors",
505
- "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors",
506
- "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
507
- "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
508
- "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
509
- "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
510
- "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors",
511
- "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors",
512
- "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors",
513
- "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors",
514
- "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors",
515
- "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors",
516
- "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors",
517
- "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors",
518
- "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
519
- "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
520
- "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
521
- "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
522
- "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors",
523
- "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors",
524
- "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors",
525
- "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors",
526
- "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors",
527
- "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors",
528
- "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors",
529
- "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors",
530
- "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
531
- "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
532
- "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
533
- "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
534
- "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors",
535
- "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors",
536
- "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors",
537
- "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors",
538
- "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors",
539
- "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors",
540
- "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors",
541
- "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors",
542
- "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
543
- "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
544
- "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
545
- "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
546
- "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors",
547
- "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors",
548
- "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors",
549
- "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors",
550
- "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors",
551
- "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors",
552
- "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors",
553
- "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors",
554
- "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
555
- "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
556
- "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
557
- "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
558
- "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors",
559
- "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors",
560
- "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors",
561
- "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors",
562
- "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors",
563
- "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors",
564
- "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors",
565
- "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors",
566
- "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
567
- "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
568
- "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
569
- "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
570
- "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors",
571
- "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors",
572
- "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors",
573
- "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors",
574
- "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors",
575
- "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors",
576
- "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors",
577
- "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors",
578
- "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
579
- "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
580
- "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
581
- "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
582
- "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors",
583
- "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors",
584
- "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors",
585
- "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors",
586
- "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors",
587
- "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors",
588
- "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors",
589
- "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors",
590
- "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
591
- "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
592
- "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
593
- "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
594
- "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors",
595
- "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors",
596
- "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors",
597
- "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors",
598
- "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors",
599
- "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors",
600
- "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors",
601
- "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors",
602
- "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
603
- "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
604
- "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
605
- "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
606
- "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors",
607
- "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors",
608
- "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors",
609
- "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors",
610
- "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors",
611
- "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors",
612
- "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors",
613
- "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors",
614
- "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
615
- "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
616
- "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
617
- "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
618
- "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors",
619
- "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors",
620
- "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors",
621
- "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors",
622
- "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors",
623
- "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors",
624
- "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors",
625
- "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors",
626
- "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
627
- "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
628
- "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
629
- "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
630
- "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors",
631
- "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors",
632
- "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors",
633
- "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors",
634
- "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors",
635
- "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors",
636
- "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors",
637
- "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors",
638
- "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
639
- "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
640
- "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
641
- "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
642
- "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors",
643
- "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors",
644
- "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors",
645
- "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors",
646
- "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors",
647
- "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors",
648
- "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors",
649
- "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors",
650
- "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
651
- "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
652
- "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
653
- "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
654
- "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors",
655
- "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors",
656
- "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors",
657
- "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors",
658
- "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors",
659
- "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors",
660
- "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors",
661
- "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors",
662
- "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
663
- "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
664
- "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
665
- "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
666
- "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors",
667
- "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors",
668
- "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors",
669
- "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors",
670
- "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors",
671
- "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors",
672
- "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors",
673
- "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors",
674
- "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
675
- "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
676
- "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
677
- "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
678
- "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors",
679
- "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors",
680
- "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors",
681
- "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors",
682
- "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors",
683
- "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors",
684
- "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors",
685
- "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors",
686
- "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
687
- "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
688
- "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
689
- "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
690
- "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors",
691
- "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors",
692
- "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors",
693
- "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors",
694
- "model.visual.deepstack_merger_list.0.linear_fc1.bias": "model-00001-of-00002.safetensors",
695
- "model.visual.deepstack_merger_list.0.linear_fc1.weight": "model-00001-of-00002.safetensors",
696
- "model.visual.deepstack_merger_list.0.linear_fc2.bias": "model-00001-of-00002.safetensors",
697
- "model.visual.deepstack_merger_list.0.linear_fc2.weight": "model-00001-of-00002.safetensors",
698
- "model.visual.deepstack_merger_list.0.norm.bias": "model-00001-of-00002.safetensors",
699
- "model.visual.deepstack_merger_list.0.norm.weight": "model-00001-of-00002.safetensors",
700
- "model.visual.deepstack_merger_list.1.linear_fc1.bias": "model-00001-of-00002.safetensors",
701
- "model.visual.deepstack_merger_list.1.linear_fc1.weight": "model-00001-of-00002.safetensors",
702
- "model.visual.deepstack_merger_list.1.linear_fc2.bias": "model-00001-of-00002.safetensors",
703
- "model.visual.deepstack_merger_list.1.linear_fc2.weight": "model-00001-of-00002.safetensors",
704
- "model.visual.deepstack_merger_list.1.norm.bias": "model-00001-of-00002.safetensors",
705
- "model.visual.deepstack_merger_list.1.norm.weight": "model-00001-of-00002.safetensors",
706
- "model.visual.deepstack_merger_list.2.linear_fc1.bias": "model-00001-of-00002.safetensors",
707
- "model.visual.deepstack_merger_list.2.linear_fc1.weight": "model-00001-of-00002.safetensors",
708
- "model.visual.deepstack_merger_list.2.linear_fc2.bias": "model-00001-of-00002.safetensors",
709
- "model.visual.deepstack_merger_list.2.linear_fc2.weight": "model-00001-of-00002.safetensors",
710
- "model.visual.deepstack_merger_list.2.norm.bias": "model-00001-of-00002.safetensors",
711
- "model.visual.deepstack_merger_list.2.norm.weight": "model-00001-of-00002.safetensors",
712
- "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors",
713
- "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors",
714
- "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors",
715
- "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors",
716
- "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors",
717
- "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors",
718
- "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors",
719
- "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors",
720
- "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors"
721
- }
722
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/preprocessor_config.json DELETED
@@ -1,39 +0,0 @@
1
- {
2
- "crop_size": null,
3
- "data_format": "channels_first",
4
- "default_to_square": true,
5
- "device": null,
6
- "disable_grouping": null,
7
- "do_center_crop": null,
8
- "do_convert_rgb": true,
9
- "do_normalize": true,
10
- "do_pad": null,
11
- "do_rescale": true,
12
- "do_resize": true,
13
- "image_mean": [
14
- 0.5,
15
- 0.5,
16
- 0.5
17
- ],
18
- "image_processor_type": "Qwen2VLImageProcessorFast",
19
- "image_std": [
20
- 0.5,
21
- 0.5,
22
- 0.5
23
- ],
24
- "input_data_format": null,
25
- "max_pixels": 1024,
26
- "merge_size": 2,
27
- "min_pixels": 1024,
28
- "pad_size": null,
29
- "patch_size": 16,
30
- "processor_class": "Qwen3VLProcessor",
31
- "resample": 3,
32
- "rescale_factor": 0.00392156862745098,
33
- "return_tensors": null,
34
- "size": {
35
- "longest_edge": 16777216,
36
- "shortest_edge": 65536
37
- },
38
- "temporal_patch_size": 2
39
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d355e48f036ee809fe47cea3a966dfaac74e37c5bb93d7478856f64a9ef76ef2
3
- size 1465
 
 
 
 
checkpoint-2113/special_tokens_map.json DELETED
@@ -1,31 +0,0 @@
1
- {
2
- "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>",
5
- "<|object_ref_start|>",
6
- "<|object_ref_end|>",
7
- "<|box_start|>",
8
- "<|box_end|>",
9
- "<|quad_start|>",
10
- "<|quad_end|>",
11
- "<|vision_start|>",
12
- "<|vision_end|>",
13
- "<|vision_pad|>",
14
- "<|image_pad|>",
15
- "<|video_pad|>"
16
- ],
17
- "eos_token": {
18
- "content": "<|im_end|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- },
24
- "pad_token": {
25
- "content": "<|endoftext|>",
26
- "lstrip": false,
27
- "normalized": false,
28
- "rstrip": false,
29
- "single_word": false
30
- }
31
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
3
- size 11422654
 
 
 
 
checkpoint-2113/tokenizer_config.json DELETED
@@ -1,242 +0,0 @@
1
- {
2
- "add_bos_token": false,
3
- "add_prefix_space": false,
4
- "added_tokens_decoder": {
5
- "151643": {
6
- "content": "<|endoftext|>",
7
- "lstrip": false,
8
- "normalized": false,
9
- "rstrip": false,
10
- "single_word": false,
11
- "special": true
12
- },
13
- "151644": {
14
- "content": "<|im_start|>",
15
- "lstrip": false,
16
- "normalized": false,
17
- "rstrip": false,
18
- "single_word": false,
19
- "special": true
20
- },
21
- "151645": {
22
- "content": "<|im_end|>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false,
27
- "special": true
28
- },
29
- "151646": {
30
- "content": "<|object_ref_start|>",
31
- "lstrip": false,
32
- "normalized": false,
33
- "rstrip": false,
34
- "single_word": false,
35
- "special": true
36
- },
37
- "151647": {
38
- "content": "<|object_ref_end|>",
39
- "lstrip": false,
40
- "normalized": false,
41
- "rstrip": false,
42
- "single_word": false,
43
- "special": true
44
- },
45
- "151648": {
46
- "content": "<|box_start|>",
47
- "lstrip": false,
48
- "normalized": false,
49
- "rstrip": false,
50
- "single_word": false,
51
- "special": true
52
- },
53
- "151649": {
54
- "content": "<|box_end|>",
55
- "lstrip": false,
56
- "normalized": false,
57
- "rstrip": false,
58
- "single_word": false,
59
- "special": true
60
- },
61
- "151650": {
62
- "content": "<|quad_start|>",
63
- "lstrip": false,
64
- "normalized": false,
65
- "rstrip": false,
66
- "single_word": false,
67
- "special": true
68
- },
69
- "151651": {
70
- "content": "<|quad_end|>",
71
- "lstrip": false,
72
- "normalized": false,
73
- "rstrip": false,
74
- "single_word": false,
75
- "special": true
76
- },
77
- "151652": {
78
- "content": "<|vision_start|>",
79
- "lstrip": false,
80
- "normalized": false,
81
- "rstrip": false,
82
- "single_word": false,
83
- "special": true
84
- },
85
- "151653": {
86
- "content": "<|vision_end|>",
87
- "lstrip": false,
88
- "normalized": false,
89
- "rstrip": false,
90
- "single_word": false,
91
- "special": true
92
- },
93
- "151654": {
94
- "content": "<|vision_pad|>",
95
- "lstrip": false,
96
- "normalized": false,
97
- "rstrip": false,
98
- "single_word": false,
99
- "special": true
100
- },
101
- "151655": {
102
- "content": "<|image_pad|>",
103
- "lstrip": false,
104
- "normalized": false,
105
- "rstrip": false,
106
- "single_word": false,
107
- "special": true
108
- },
109
- "151656": {
110
- "content": "<|video_pad|>",
111
- "lstrip": false,
112
- "normalized": false,
113
- "rstrip": false,
114
- "single_word": false,
115
- "special": true
116
- },
117
- "151657": {
118
- "content": "<tool_call>",
119
- "lstrip": false,
120
- "normalized": false,
121
- "rstrip": false,
122
- "single_word": false,
123
- "special": false
124
- },
125
- "151658": {
126
- "content": "</tool_call>",
127
- "lstrip": false,
128
- "normalized": false,
129
- "rstrip": false,
130
- "single_word": false,
131
- "special": false
132
- },
133
- "151659": {
134
- "content": "<|fim_prefix|>",
135
- "lstrip": false,
136
- "normalized": false,
137
- "rstrip": false,
138
- "single_word": false,
139
- "special": false
140
- },
141
- "151660": {
142
- "content": "<|fim_middle|>",
143
- "lstrip": false,
144
- "normalized": false,
145
- "rstrip": false,
146
- "single_word": false,
147
- "special": false
148
- },
149
- "151661": {
150
- "content": "<|fim_suffix|>",
151
- "lstrip": false,
152
- "normalized": false,
153
- "rstrip": false,
154
- "single_word": false,
155
- "special": false
156
- },
157
- "151662": {
158
- "content": "<|fim_pad|>",
159
- "lstrip": false,
160
- "normalized": false,
161
- "rstrip": false,
162
- "single_word": false,
163
- "special": false
164
- },
165
- "151663": {
166
- "content": "<|repo_name|>",
167
- "lstrip": false,
168
- "normalized": false,
169
- "rstrip": false,
170
- "single_word": false,
171
- "special": false
172
- },
173
- "151664": {
174
- "content": "<|file_sep|>",
175
- "lstrip": false,
176
- "normalized": false,
177
- "rstrip": false,
178
- "single_word": false,
179
- "special": false
180
- },
181
- "151665": {
182
- "content": "<tool_response>",
183
- "lstrip": false,
184
- "normalized": false,
185
- "rstrip": false,
186
- "single_word": false,
187
- "special": false
188
- },
189
- "151666": {
190
- "content": "</tool_response>",
191
- "lstrip": false,
192
- "normalized": false,
193
- "rstrip": false,
194
- "single_word": false,
195
- "special": false
196
- },
197
- "151667": {
198
- "content": "<think>",
199
- "lstrip": false,
200
- "normalized": false,
201
- "rstrip": false,
202
- "single_word": false,
203
- "special": false
204
- },
205
- "151668": {
206
- "content": "</think>",
207
- "lstrip": false,
208
- "normalized": false,
209
- "rstrip": false,
210
- "single_word": false,
211
- "special": false
212
- }
213
- },
214
- "additional_special_tokens": [
215
- "<|im_start|>",
216
- "<|im_end|>",
217
- "<|object_ref_start|>",
218
- "<|object_ref_end|>",
219
- "<|box_start|>",
220
- "<|box_end|>",
221
- "<|quad_start|>",
222
- "<|quad_end|>",
223
- "<|vision_start|>",
224
- "<|vision_end|>",
225
- "<|vision_pad|>",
226
- "<|image_pad|>",
227
- "<|video_pad|>"
228
- ],
229
- "bos_token": null,
230
- "clean_up_tokenization_spaces": false,
231
- "eos_token": "<|im_end|>",
232
- "errors": "replace",
233
- "extra_special_tokens": {},
234
- "max_pixels": 1024,
235
- "min_pixels": 1024,
236
- "model_max_length": 1024,
237
- "pad_token": "<|endoftext|>",
238
- "processor_class": "Qwen3VLProcessor",
239
- "split_special_tokens": false,
240
- "tokenizer_class": "Qwen2Tokenizer",
241
- "unk_token": null
242
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/trainer_state.json DELETED
@@ -1,2144 +0,0 @@
1
- {
2
- "best_global_step": null,
3
- "best_metric": null,
4
- "best_model_checkpoint": null,
5
- "epoch": 1.0,
6
- "eval_steps": 500,
7
- "global_step": 2113,
8
- "is_hyper_param_search": false,
9
- "is_local_process_zero": true,
10
- "is_world_process_zero": true,
11
- "log_history": [
12
- {
13
- "entropy": 1.9664676249027253,
14
- "epoch": 0.004733167672464797,
15
- "grad_norm": 2.9172720909118652,
16
- "learning_rate": 8.490566037735849e-07,
17
- "loss": 2.4508,
18
- "mean_token_accuracy": 0.5263931974768639,
19
- "num_tokens": 3879417.0,
20
- "step": 10
21
- },
22
- {
23
- "entropy": 2.0464825958013533,
24
- "epoch": 0.009466335344929595,
25
- "grad_norm": 0.7783076763153076,
26
- "learning_rate": 1.7924528301886793e-06,
27
- "loss": 2.2632,
28
- "mean_token_accuracy": 0.5380982637405396,
29
- "num_tokens": 7799425.0,
30
- "step": 20
31
- },
32
- {
33
- "entropy": 2.0156721025705338,
34
- "epoch": 0.014199503017394392,
35
- "grad_norm": 0.38891270756721497,
36
- "learning_rate": 2.7358490566037738e-06,
37
- "loss": 2.0189,
38
- "mean_token_accuracy": 0.568930983543396,
39
- "num_tokens": 11646570.0,
40
- "step": 30
41
- },
42
- {
43
- "entropy": 1.8222261607646941,
44
- "epoch": 0.01893267068985919,
45
- "grad_norm": 0.2623351812362671,
46
- "learning_rate": 3.679245283018868e-06,
47
- "loss": 1.8679,
48
- "mean_token_accuracy": 0.5922655925154686,
49
- "num_tokens": 15549095.0,
50
- "step": 40
51
- },
52
- {
53
- "entropy": 1.7781452298164369,
54
- "epoch": 0.023665838362323985,
55
- "grad_norm": 0.19355912506580353,
56
- "learning_rate": 4.622641509433963e-06,
57
- "loss": 1.7967,
58
- "mean_token_accuracy": 0.6030044049024582,
59
- "num_tokens": 19395583.0,
60
- "step": 50
61
- },
62
- {
63
- "entropy": 1.756221604347229,
64
- "epoch": 0.028399006034788784,
65
- "grad_norm": 0.17072290182113647,
66
- "learning_rate": 5.566037735849057e-06,
67
- "loss": 1.7705,
68
- "mean_token_accuracy": 0.6082265704870224,
69
- "num_tokens": 23287255.0,
70
- "step": 60
71
- },
72
- {
73
- "entropy": 1.721426972746849,
74
- "epoch": 0.033132173707253576,
75
- "grad_norm": 0.18896205723285675,
76
- "learning_rate": 6.5094339622641515e-06,
77
- "loss": 1.7402,
78
- "mean_token_accuracy": 0.6131706237792969,
79
- "num_tokens": 27155383.0,
80
- "step": 70
81
- },
82
- {
83
- "entropy": 1.720760390162468,
84
- "epoch": 0.03786534137971838,
85
- "grad_norm": 0.16523988544940948,
86
- "learning_rate": 7.452830188679246e-06,
87
- "loss": 1.7276,
88
- "mean_token_accuracy": 0.615737484395504,
89
- "num_tokens": 31071816.0,
90
- "step": 80
91
- },
92
- {
93
- "entropy": 1.696010684967041,
94
- "epoch": 0.042598509052183174,
95
- "grad_norm": 0.17002570629119873,
96
- "learning_rate": 8.39622641509434e-06,
97
- "loss": 1.7075,
98
- "mean_token_accuracy": 0.6187006324529648,
99
- "num_tokens": 34979450.0,
100
- "step": 90
101
- },
102
- {
103
- "entropy": 1.6817427426576614,
104
- "epoch": 0.04733167672464797,
105
- "grad_norm": 0.1671409159898758,
106
- "learning_rate": 9.339622641509435e-06,
107
- "loss": 1.689,
108
- "mean_token_accuracy": 0.6216033041477204,
109
- "num_tokens": 38828299.0,
110
- "step": 100
111
- },
112
- {
113
- "entropy": 1.6723841279745102,
114
- "epoch": 0.052064844397112765,
115
- "grad_norm": 0.1683770716190338,
116
- "learning_rate": 9.999944870161476e-06,
117
- "loss": 1.683,
118
- "mean_token_accuracy": 0.6222904548048973,
119
- "num_tokens": 42710858.0,
120
- "step": 110
121
- },
122
- {
123
- "entropy": 1.6607272058725357,
124
- "epoch": 0.05679801206957757,
125
- "grad_norm": 0.16289345920085907,
126
- "learning_rate": 9.998964817962902e-06,
127
- "loss": 1.6697,
128
- "mean_token_accuracy": 0.6255089312791824,
129
- "num_tokens": 46563041.0,
130
- "step": 120
131
- },
132
- {
133
- "entropy": 1.6530560374259948,
134
- "epoch": 0.06153117974204236,
135
- "grad_norm": 0.1647324562072754,
136
- "learning_rate": 9.996759934642046e-06,
137
- "loss": 1.66,
138
- "mean_token_accuracy": 0.6271017372608185,
139
- "num_tokens": 50473774.0,
140
- "step": 130
141
- },
142
- {
143
- "entropy": 1.646892574429512,
144
- "epoch": 0.06626434741450715,
145
- "grad_norm": 0.1683024764060974,
146
- "learning_rate": 9.993330760432702e-06,
147
- "loss": 1.6506,
148
- "mean_token_accuracy": 0.6288532495498658,
149
- "num_tokens": 54345703.0,
150
- "step": 140
151
- },
152
- {
153
- "entropy": 1.6421253114938736,
154
- "epoch": 0.07099751508697195,
155
- "grad_norm": 0.16962556540966034,
156
- "learning_rate": 9.98867813554068e-06,
157
- "loss": 1.649,
158
- "mean_token_accuracy": 0.6289903283119201,
159
- "num_tokens": 58224397.0,
160
- "step": 150
161
- },
162
- {
163
- "entropy": 1.6290255516767502,
164
- "epoch": 0.07573068275943676,
165
- "grad_norm": 0.16403813660144806,
166
- "learning_rate": 9.98280319993795e-06,
167
- "loss": 1.6341,
168
- "mean_token_accuracy": 0.6317014634609223,
169
- "num_tokens": 62114712.0,
170
- "step": 160
171
- },
172
- {
173
- "entropy": 1.6350149989128113,
174
- "epoch": 0.08046385043190155,
175
- "grad_norm": 0.16714808344841003,
176
- "learning_rate": 9.975707393083328e-06,
177
- "loss": 1.6379,
178
- "mean_token_accuracy": 0.6297940164804459,
179
- "num_tokens": 65991061.0,
180
- "step": 170
181
- },
182
- {
183
- "entropy": 1.6163844734430313,
184
- "epoch": 0.08519701810436635,
185
- "grad_norm": 0.17177364230155945,
186
- "learning_rate": 9.967392453569775e-06,
187
- "loss": 1.6222,
188
- "mean_token_accuracy": 0.6339128240942955,
189
- "num_tokens": 69856575.0,
190
- "step": 180
191
- },
192
- {
193
- "entropy": 1.6292197763919831,
194
- "epoch": 0.08993018577683115,
195
- "grad_norm": 0.16445177793502808,
196
- "learning_rate": 9.95786041869843e-06,
197
- "loss": 1.6316,
198
- "mean_token_accuracy": 0.6325667470693588,
199
- "num_tokens": 73718253.0,
200
- "step": 190
201
- },
202
- {
203
- "entropy": 1.6217930316925049,
204
- "epoch": 0.09466335344929594,
205
- "grad_norm": 0.15951167047023773,
206
- "learning_rate": 9.947113623979423e-06,
207
- "loss": 1.6253,
208
- "mean_token_accuracy": 0.6325555801391601,
209
- "num_tokens": 77599124.0,
210
- "step": 200
211
- },
212
- {
213
- "entropy": 1.6170487314462663,
214
- "epoch": 0.09939652112176074,
215
- "grad_norm": 0.15958699584007263,
216
- "learning_rate": 9.935154702559637e-06,
217
- "loss": 1.6246,
218
- "mean_token_accuracy": 0.6333600997924804,
219
- "num_tokens": 81539199.0,
220
- "step": 210
221
- },
222
- {
223
- "entropy": 1.6078982055187225,
224
- "epoch": 0.10412968879422553,
225
- "grad_norm": 0.16408687829971313,
226
- "learning_rate": 9.921986584577545e-06,
227
- "loss": 1.6135,
228
- "mean_token_accuracy": 0.6353577077388763,
229
- "num_tokens": 85404474.0,
230
- "step": 220
231
- },
232
- {
233
- "entropy": 1.6178746074438095,
234
- "epoch": 0.10886285646669033,
235
- "grad_norm": 0.16022811830043793,
236
- "learning_rate": 9.90761249644528e-06,
237
- "loss": 1.6173,
238
- "mean_token_accuracy": 0.6340486526489257,
239
- "num_tokens": 89310569.0,
240
- "step": 230
241
- },
242
- {
243
- "entropy": 1.6118096917867661,
244
- "epoch": 0.11359602413915514,
245
- "grad_norm": 0.16375407576560974,
246
- "learning_rate": 9.8920359600581e-06,
247
- "loss": 1.6195,
248
- "mean_token_accuracy": 0.6345633432269097,
249
- "num_tokens": 93225240.0,
250
- "step": 240
251
- },
252
- {
253
- "entropy": 1.6142515003681184,
254
- "epoch": 0.11832919181161992,
255
- "grad_norm": 0.16436107456684113,
256
- "learning_rate": 9.875260791931474e-06,
257
- "loss": 1.6154,
258
- "mean_token_accuracy": 0.6353823691606522,
259
- "num_tokens": 97106446.0,
260
- "step": 250
261
- },
262
- {
263
- "entropy": 1.6073731541633607,
264
- "epoch": 0.12306235948408473,
265
- "grad_norm": 0.16237056255340576,
266
- "learning_rate": 9.85729110226596e-06,
267
- "loss": 1.6087,
268
- "mean_token_accuracy": 0.6353139713406563,
269
- "num_tokens": 100977975.0,
270
- "step": 260
271
- },
272
- {
273
- "entropy": 1.6020812511444091,
274
- "epoch": 0.12779552715654952,
275
- "grad_norm": 0.15944795310497284,
276
- "learning_rate": 9.838131293940148e-06,
277
- "loss": 1.6071,
278
- "mean_token_accuracy": 0.6363679498434067,
279
- "num_tokens": 104876425.0,
280
- "step": 270
281
- },
282
- {
283
- "entropy": 1.6033865660429,
284
- "epoch": 0.1325286948290143,
285
- "grad_norm": 0.17549684643745422,
286
- "learning_rate": 9.817786061431874e-06,
287
- "loss": 1.6091,
288
- "mean_token_accuracy": 0.6354461714625359,
289
- "num_tokens": 108709842.0,
290
- "step": 280
291
- },
292
- {
293
- "entropy": 1.5905151784420013,
294
- "epoch": 0.13726186250147912,
295
- "grad_norm": 0.1742265522480011,
296
- "learning_rate": 9.79626038966799e-06,
297
- "loss": 1.596,
298
- "mean_token_accuracy": 0.6382940918207168,
299
- "num_tokens": 112609412.0,
300
- "step": 290
301
- },
302
- {
303
- "entropy": 1.5923225700855255,
304
- "epoch": 0.1419950301739439,
305
- "grad_norm": 0.16349110007286072,
306
- "learning_rate": 9.773559552802982e-06,
307
- "loss": 1.5926,
308
- "mean_token_accuracy": 0.6394994035363197,
309
- "num_tokens": 116498424.0,
310
- "step": 300
311
- },
312
- {
313
- "entropy": 1.5896567821502685,
314
- "epoch": 0.1467281978464087,
315
- "grad_norm": 0.1612998992204666,
316
- "learning_rate": 9.749689112926701e-06,
317
- "loss": 1.5973,
318
- "mean_token_accuracy": 0.6379290148615837,
319
- "num_tokens": 120371695.0,
320
- "step": 310
321
- },
322
- {
323
- "entropy": 1.596482390165329,
324
- "epoch": 0.15146136551887351,
325
- "grad_norm": 0.16588379442691803,
326
- "learning_rate": 9.724654918701568e-06,
327
- "loss": 1.6006,
328
- "mean_token_accuracy": 0.6373828530311585,
329
- "num_tokens": 124255939.0,
330
- "step": 320
331
- },
332
- {
333
- "entropy": 1.584341713786125,
334
- "epoch": 0.1561945331913383,
335
- "grad_norm": 0.16518791019916534,
336
- "learning_rate": 9.698463103929542e-06,
337
- "loss": 1.5887,
338
- "mean_token_accuracy": 0.6403212890028953,
339
- "num_tokens": 128177070.0,
340
- "step": 330
341
- },
342
- {
343
- "entropy": 1.5725542455911636,
344
- "epoch": 0.1609277008638031,
345
- "grad_norm": 0.16715767979621887,
346
- "learning_rate": 9.671120086049246e-06,
347
- "loss": 1.5799,
348
- "mean_token_accuracy": 0.6416747942566872,
349
- "num_tokens": 132121810.0,
350
- "step": 340
351
- },
352
- {
353
- "entropy": 1.5927457183599472,
354
- "epoch": 0.1656608685362679,
355
- "grad_norm": 0.17323796451091766,
356
- "learning_rate": 9.642632564563576e-06,
357
- "loss": 1.5939,
358
- "mean_token_accuracy": 0.6393081307411194,
359
- "num_tokens": 136012804.0,
360
- "step": 350
361
- },
362
- {
363
- "entropy": 1.5822071313858033,
364
- "epoch": 0.1703940362087327,
365
- "grad_norm": 0.16068372130393982,
366
- "learning_rate": 9.61300751939821e-06,
367
- "loss": 1.583,
368
- "mean_token_accuracy": 0.6409055680036545,
369
- "num_tokens": 139940284.0,
370
- "step": 360
371
- },
372
- {
373
- "entropy": 1.568736445903778,
374
- "epoch": 0.17512720388119749,
375
- "grad_norm": 0.16828852891921997,
376
- "learning_rate": 9.582252209191417e-06,
377
- "loss": 1.5788,
378
- "mean_token_accuracy": 0.6417408421635628,
379
- "num_tokens": 143849853.0,
380
- "step": 370
381
- },
382
- {
383
- "entropy": 1.5762336939573287,
384
- "epoch": 0.1798603715536623,
385
- "grad_norm": 0.1591760367155075,
386
- "learning_rate": 9.550374169515557e-06,
387
- "loss": 1.5863,
388
- "mean_token_accuracy": 0.6399773553013801,
389
- "num_tokens": 147708328.0,
390
- "step": 380
391
- },
392
- {
393
- "entropy": 1.5851975083351135,
394
- "epoch": 0.1845935392261271,
395
- "grad_norm": 0.16111813485622406,
396
- "learning_rate": 9.517381211030745e-06,
397
- "loss": 1.5859,
398
- "mean_token_accuracy": 0.6406419515609741,
399
- "num_tokens": 151541012.0,
400
- "step": 390
401
- },
402
- {
403
- "entropy": 1.5762753903865814,
404
- "epoch": 0.18932670689859188,
405
- "grad_norm": 0.1650865077972412,
406
- "learning_rate": 9.4832814175711e-06,
407
- "loss": 1.583,
408
- "mean_token_accuracy": 0.6416757836937904,
409
- "num_tokens": 155434817.0,
410
- "step": 400
411
- },
412
- {
413
- "entropy": 1.5776998460292817,
414
- "epoch": 0.19405987457105667,
415
- "grad_norm": 0.1527196615934372,
416
- "learning_rate": 9.448083144164078e-06,
417
- "loss": 1.5798,
418
- "mean_token_accuracy": 0.6412320747971535,
419
- "num_tokens": 159311737.0,
420
- "step": 410
421
- },
422
- {
423
- "entropy": 1.5809161812067032,
424
- "epoch": 0.19879304224352148,
425
- "grad_norm": 0.16029658913612366,
426
- "learning_rate": 9.411795014983338e-06,
427
- "loss": 1.5847,
428
- "mean_token_accuracy": 0.64069664478302,
429
- "num_tokens": 163185815.0,
430
- "step": 420
431
- },
432
- {
433
- "entropy": 1.582759228348732,
434
- "epoch": 0.20352620991598627,
435
- "grad_norm": 0.15568074584007263,
436
- "learning_rate": 9.374425921235684e-06,
437
- "loss": 1.5865,
438
- "mean_token_accuracy": 0.6403465285897255,
439
- "num_tokens": 167027904.0,
440
- "step": 430
441
- },
442
- {
443
- "entropy": 1.5735624372959136,
444
- "epoch": 0.20825937758845106,
445
- "grad_norm": 0.15200895071029663,
446
- "learning_rate": 9.33598501898256e-06,
447
- "loss": 1.5796,
448
- "mean_token_accuracy": 0.6411942645907402,
449
- "num_tokens": 170922541.0,
450
- "step": 440
451
- },
452
- {
453
- "entropy": 1.573996353149414,
454
- "epoch": 0.21299254526091588,
455
- "grad_norm": 0.15826307237148285,
456
- "learning_rate": 9.296481726896663e-06,
457
- "loss": 1.5784,
458
- "mean_token_accuracy": 0.6419912785291672,
459
- "num_tokens": 174791554.0,
460
- "step": 450
461
- },
462
- {
463
- "entropy": 1.573242038488388,
464
- "epoch": 0.21772571293338067,
465
- "grad_norm": 0.16425147652626038,
466
- "learning_rate": 9.25592572395421e-06,
467
- "loss": 1.5741,
468
- "mean_token_accuracy": 0.6426722049713135,
469
- "num_tokens": 178691330.0,
470
- "step": 460
471
- },
472
- {
473
- "entropy": 1.5621694713830947,
474
- "epoch": 0.22245888060584545,
475
- "grad_norm": 0.15976452827453613,
476
- "learning_rate": 9.214326947063424e-06,
477
- "loss": 1.569,
478
- "mean_token_accuracy": 0.6429914653301239,
479
- "num_tokens": 182540865.0,
480
- "step": 470
481
- },
482
- {
483
- "entropy": 1.5637736171483994,
484
- "epoch": 0.22719204827831027,
485
- "grad_norm": 0.15400023758411407,
486
- "learning_rate": 9.171695588629818e-06,
487
- "loss": 1.5669,
488
- "mean_token_accuracy": 0.6447574943304062,
489
- "num_tokens": 186445682.0,
490
- "step": 480
491
- },
492
- {
493
- "entropy": 1.5614915072917939,
494
- "epoch": 0.23192521595077506,
495
- "grad_norm": 0.1612849235534668,
496
- "learning_rate": 9.128042094058892e-06,
497
- "loss": 1.5661,
498
- "mean_token_accuracy": 0.6434317573904991,
499
- "num_tokens": 190298343.0,
500
- "step": 490
501
- },
502
- {
503
- "entropy": 1.57402603328228,
504
- "epoch": 0.23665838362323985,
505
- "grad_norm": 0.1595270037651062,
506
- "learning_rate": 9.083377159196824e-06,
507
- "loss": 1.5783,
508
- "mean_token_accuracy": 0.6416443169116974,
509
- "num_tokens": 194193930.0,
510
- "step": 500
511
- },
512
- {
513
- "entropy": 1.5727292507886887,
514
- "epoch": 0.24139155129570464,
515
- "grad_norm": 0.1542680561542511,
516
- "learning_rate": 9.037711727709812e-06,
517
- "loss": 1.5708,
518
- "mean_token_accuracy": 0.6426661461591721,
519
- "num_tokens": 198099362.0,
520
- "step": 510
521
- },
522
- {
523
- "entropy": 1.5556272625923158,
524
- "epoch": 0.24612471896816945,
525
- "grad_norm": 0.16021884977817535,
526
- "learning_rate": 8.991056988402682e-06,
527
- "loss": 1.5594,
528
- "mean_token_accuracy": 0.6437688574194909,
529
- "num_tokens": 201971348.0,
530
- "step": 520
531
- },
532
- {
533
- "entropy": 1.561934667825699,
534
- "epoch": 0.25085788664063424,
535
- "grad_norm": 0.160409614443779,
536
- "learning_rate": 8.943424372477455e-06,
537
- "loss": 1.5663,
538
- "mean_token_accuracy": 0.6441657856106758,
539
- "num_tokens": 205887553.0,
540
- "step": 530
541
- },
542
- {
543
- "entropy": 1.5650932669639588,
544
- "epoch": 0.25559105431309903,
545
- "grad_norm": 0.16672302782535553,
546
- "learning_rate": 8.894825550732491e-06,
547
- "loss": 1.5651,
548
- "mean_token_accuracy": 0.6445234566926956,
549
- "num_tokens": 209773471.0,
550
- "step": 540
551
- },
552
- {
553
- "entropy": 1.5703383475542068,
554
- "epoch": 0.2603242219855638,
555
- "grad_norm": 0.1647508144378662,
556
- "learning_rate": 8.845272430702962e-06,
557
- "loss": 1.5719,
558
- "mean_token_accuracy": 0.6425846204161644,
559
- "num_tokens": 213667761.0,
560
- "step": 550
561
- },
562
- {
563
- "entropy": 1.5690355956554414,
564
- "epoch": 0.2650573896580286,
565
- "grad_norm": 0.16149023175239563,
566
- "learning_rate": 8.79477715374329e-06,
567
- "loss": 1.5733,
568
- "mean_token_accuracy": 0.6426513850688934,
569
- "num_tokens": 217473103.0,
570
- "step": 560
571
- },
572
- {
573
- "entropy": 1.565832221508026,
574
- "epoch": 0.26979055733049345,
575
- "grad_norm": 0.1554195135831833,
576
- "learning_rate": 8.743352092052326e-06,
577
- "loss": 1.568,
578
- "mean_token_accuracy": 0.6430184543132782,
579
- "num_tokens": 221365501.0,
580
- "step": 570
581
- },
582
- {
583
- "entropy": 1.562074252963066,
584
- "epoch": 0.27452372500295824,
585
- "grad_norm": 0.16073721647262573,
586
- "learning_rate": 8.691009845641931e-06,
587
- "loss": 1.5701,
588
- "mean_token_accuracy": 0.6420939102768898,
589
- "num_tokens": 225235652.0,
590
- "step": 580
591
- },
592
- {
593
- "entropy": 1.5577911257743835,
594
- "epoch": 0.27925689267542303,
595
- "grad_norm": 0.16031447052955627,
596
- "learning_rate": 8.637763239249778e-06,
597
- "loss": 1.5627,
598
- "mean_token_accuracy": 0.6444457039237023,
599
- "num_tokens": 229134110.0,
600
- "step": 590
601
- },
602
- {
603
- "entropy": 1.556574183702469,
604
- "epoch": 0.2839900603478878,
605
- "grad_norm": 0.1577136367559433,
606
- "learning_rate": 8.583625319197063e-06,
607
- "loss": 1.5565,
608
- "mean_token_accuracy": 0.6455687090754509,
609
- "num_tokens": 233039906.0,
610
- "step": 600
611
- },
612
- {
613
- "entropy": 1.5509059250354766,
614
- "epoch": 0.2887232280203526,
615
- "grad_norm": 0.15498368442058563,
616
- "learning_rate": 8.528609350191938e-06,
617
- "loss": 1.558,
618
- "mean_token_accuracy": 0.6440837427973747,
619
- "num_tokens": 236877041.0,
620
- "step": 610
621
- },
622
- {
623
- "entropy": 1.5675470024347304,
624
- "epoch": 0.2934563956928174,
625
- "grad_norm": 0.16251349449157715,
626
- "learning_rate": 8.472728812079436e-06,
627
- "loss": 1.5698,
628
- "mean_token_accuracy": 0.6437425896525383,
629
- "num_tokens": 240760840.0,
630
- "step": 620
631
- },
632
- {
633
- "entropy": 1.5483269661664962,
634
- "epoch": 0.29818956336528224,
635
- "grad_norm": 0.16416430473327637,
636
- "learning_rate": 8.415997396538671e-06,
637
- "loss": 1.5516,
638
- "mean_token_accuracy": 0.6461272791028023,
639
- "num_tokens": 244646788.0,
640
- "step": 630
641
- },
642
- {
643
- "entropy": 1.5591329038143158,
644
- "epoch": 0.30292273103774703,
645
- "grad_norm": 0.15344582498073578,
646
- "learning_rate": 8.358429003728158e-06,
647
- "loss": 1.5631,
648
- "mean_token_accuracy": 0.6448241740465164,
649
- "num_tokens": 248505578.0,
650
- "step": 640
651
- },
652
- {
653
- "entropy": 1.5571163922548295,
654
- "epoch": 0.3076558987102118,
655
- "grad_norm": 0.16142602264881134,
656
- "learning_rate": 8.30003773888003e-06,
657
- "loss": 1.561,
658
- "mean_token_accuracy": 0.6446821197867394,
659
- "num_tokens": 252442387.0,
660
- "step": 650
661
- },
662
- {
663
- "entropy": 1.543628853559494,
664
- "epoch": 0.3123890663826766,
665
- "grad_norm": 0.16222691535949707,
666
- "learning_rate": 8.240837908844011e-06,
667
- "loss": 1.5465,
668
- "mean_token_accuracy": 0.6476170778274536,
669
- "num_tokens": 256347145.0,
670
- "step": 660
671
- },
672
- {
673
- "entropy": 1.5412938565015792,
674
- "epoch": 0.3171222340551414,
675
- "grad_norm": 0.1638648957014084,
676
- "learning_rate": 8.180844018582014e-06,
677
- "loss": 1.5483,
678
- "mean_token_accuracy": 0.6471631437540054,
679
- "num_tokens": 260255832.0,
680
- "step": 670
681
- },
682
- {
683
- "entropy": 1.5546195566654206,
684
- "epoch": 0.3218554017276062,
685
- "grad_norm": 0.1579921394586563,
686
- "learning_rate": 8.12007076761416e-06,
687
- "loss": 1.5539,
688
- "mean_token_accuracy": 0.6463906392455101,
689
- "num_tokens": 264145360.0,
690
- "step": 680
691
- },
692
- {
693
- "entropy": 1.538521057367325,
694
- "epoch": 0.32658856940007097,
695
- "grad_norm": 0.15421803295612335,
696
- "learning_rate": 8.05853304641716e-06,
697
- "loss": 1.5499,
698
- "mean_token_accuracy": 0.6467000633478165,
699
- "num_tokens": 268031581.0,
700
- "step": 690
701
- },
702
- {
703
- "entropy": 1.5507268905639648,
704
- "epoch": 0.3313217370725358,
705
- "grad_norm": 0.16604824364185333,
706
- "learning_rate": 7.996245932775883e-06,
707
- "loss": 1.5511,
708
- "mean_token_accuracy": 0.6466175228357315,
709
- "num_tokens": 271952881.0,
710
- "step": 700
711
- },
712
- {
713
- "entropy": 1.5473167181015015,
714
- "epoch": 0.3360549047450006,
715
- "grad_norm": 0.16262564063072205,
716
- "learning_rate": 7.933224688089059e-06,
717
- "loss": 1.5454,
718
- "mean_token_accuracy": 0.6477824598550797,
719
- "num_tokens": 275858331.0,
720
- "step": 710
721
- },
722
- {
723
- "entropy": 1.5481310993433,
724
- "epoch": 0.3407880724174654,
725
- "grad_norm": 0.1532575786113739,
726
- "learning_rate": 7.869484753629963e-06,
727
- "loss": 1.5541,
728
- "mean_token_accuracy": 0.6460428267717362,
729
- "num_tokens": 279739647.0,
730
- "step": 720
731
- },
732
- {
733
- "entropy": 1.5434869408607483,
734
- "epoch": 0.3455212400899302,
735
- "grad_norm": 0.1606515794992447,
736
- "learning_rate": 7.805041746763052e-06,
737
- "loss": 1.5495,
738
- "mean_token_accuracy": 0.6475459963083268,
739
- "num_tokens": 283656822.0,
740
- "step": 730
741
- },
742
- {
743
- "entropy": 1.550658145546913,
744
- "epoch": 0.35025440776239497,
745
- "grad_norm": 0.16676083207130432,
746
- "learning_rate": 7.739911457117437e-06,
747
- "loss": 1.5461,
748
- "mean_token_accuracy": 0.6471642255783081,
749
- "num_tokens": 287491081.0,
750
- "step": 740
751
- },
752
- {
753
- "entropy": 1.5445130974054337,
754
- "epoch": 0.35498757543485976,
755
- "grad_norm": 0.15253710746765137,
756
- "learning_rate": 7.674109842718163e-06,
757
- "loss": 1.5545,
758
- "mean_token_accuracy": 0.646132855117321,
759
- "num_tokens": 291367807.0,
760
- "step": 750
761
- },
762
- {
763
- "entropy": 1.5350563913583755,
764
- "epoch": 0.3597207431073246,
765
- "grad_norm": 0.16385188698768616,
766
- "learning_rate": 7.607653026076218e-06,
767
- "loss": 1.5395,
768
- "mean_token_accuracy": 0.6482170835137367,
769
- "num_tokens": 295210867.0,
770
- "step": 760
771
- },
772
- {
773
- "entropy": 1.5343231499195098,
774
- "epoch": 0.3644539107797894,
775
- "grad_norm": 0.1605066955089569,
776
- "learning_rate": 7.540557290238251e-06,
777
- "loss": 1.5386,
778
- "mean_token_accuracy": 0.648663455247879,
779
- "num_tokens": 299093246.0,
780
- "step": 770
781
- },
782
- {
783
- "entropy": 1.5489379942417145,
784
- "epoch": 0.3691870784522542,
785
- "grad_norm": 0.16125887632369995,
786
- "learning_rate": 7.4728390747969385e-06,
787
- "loss": 1.554,
788
- "mean_token_accuracy": 0.646386744081974,
789
- "num_tokens": 302979174.0,
790
- "step": 780
791
- },
792
- {
793
- "entropy": 1.5436400204896927,
794
- "epoch": 0.37392024612471897,
795
- "grad_norm": 0.16131755709648132,
796
- "learning_rate": 7.404514971863015e-06,
797
- "loss": 1.5547,
798
- "mean_token_accuracy": 0.6458986714482308,
799
- "num_tokens": 306861334.0,
800
- "step": 790
801
- },
802
- {
803
- "entropy": 1.5432285010814666,
804
- "epoch": 0.37865341379718376,
805
- "grad_norm": 0.15674395859241486,
806
- "learning_rate": 7.3356017219999236e-06,
807
- "loss": 1.543,
808
- "mean_token_accuracy": 0.6479764103889465,
809
- "num_tokens": 310763957.0,
810
- "step": 800
811
- },
812
- {
813
- "entropy": 1.5451003342866898,
814
- "epoch": 0.38338658146964855,
815
- "grad_norm": 0.1598990261554718,
816
- "learning_rate": 7.2661162101220895e-06,
817
- "loss": 1.5494,
818
- "mean_token_accuracy": 0.6463793829083443,
819
- "num_tokens": 314645121.0,
820
- "step": 810
821
- },
822
- {
823
- "entropy": 1.5456315219402312,
824
- "epoch": 0.38811974914211333,
825
- "grad_norm": 0.15369856357574463,
826
- "learning_rate": 7.196075461357831e-06,
827
- "loss": 1.5488,
828
- "mean_token_accuracy": 0.6465605661273003,
829
- "num_tokens": 318500691.0,
830
- "step": 820
831
- },
832
- {
833
- "entropy": 1.5482069969177246,
834
- "epoch": 0.3928529168145782,
835
- "grad_norm": 0.1560712456703186,
836
- "learning_rate": 7.125496636877922e-06,
837
- "loss": 1.5473,
838
- "mean_token_accuracy": 0.6471294403076172,
839
- "num_tokens": 322430307.0,
840
- "step": 830
841
- },
842
- {
843
- "entropy": 1.527587327361107,
844
- "epoch": 0.39758608448704297,
845
- "grad_norm": 0.154985710978508,
846
- "learning_rate": 7.054397029690802e-06,
847
- "loss": 1.5341,
848
- "mean_token_accuracy": 0.6498543590307235,
849
- "num_tokens": 326334568.0,
850
- "step": 840
851
- },
852
- {
853
- "entropy": 1.5514892965555191,
854
- "epoch": 0.40231925215950776,
855
- "grad_norm": 0.15923038125038147,
856
- "learning_rate": 6.982794060405502e-06,
857
- "loss": 1.5525,
858
- "mean_token_accuracy": 0.6457523703575134,
859
- "num_tokens": 330201587.0,
860
- "step": 850
861
- },
862
- {
863
- "entropy": 1.533634251356125,
864
- "epoch": 0.40705241983197255,
865
- "grad_norm": 0.16257378458976746,
866
- "learning_rate": 6.910705272963307e-06,
867
- "loss": 1.5412,
868
- "mean_token_accuracy": 0.6487752512097359,
869
- "num_tokens": 334119000.0,
870
- "step": 860
871
- },
872
- {
873
- "entropy": 1.5422323346138,
874
- "epoch": 0.41178558750443733,
875
- "grad_norm": 0.15663348138332367,
876
- "learning_rate": 6.8381483303391795e-06,
877
- "loss": 1.5437,
878
- "mean_token_accuracy": 0.6477985471487046,
879
- "num_tokens": 337973430.0,
880
- "step": 870
881
- },
882
- {
883
- "entropy": 1.5455356657505035,
884
- "epoch": 0.4165187551769021,
885
- "grad_norm": 0.15920013189315796,
886
- "learning_rate": 6.765141010214038e-06,
887
- "loss": 1.5461,
888
- "mean_token_accuracy": 0.6477194577455521,
889
- "num_tokens": 341848262.0,
890
- "step": 880
891
- },
892
- {
893
- "entropy": 1.5300615668296813,
894
- "epoch": 0.42125192284936697,
895
- "grad_norm": 0.15260560810565948,
896
- "learning_rate": 6.691701200618925e-06,
897
- "loss": 1.5355,
898
- "mean_token_accuracy": 0.6492922276258468,
899
- "num_tokens": 345741246.0,
900
- "step": 890
901
- },
902
- {
903
- "entropy": 1.5360501438379288,
904
- "epoch": 0.42598509052183176,
905
- "grad_norm": 0.1613568812608719,
906
- "learning_rate": 6.617846895552137e-06,
907
- "loss": 1.5409,
908
- "mean_token_accuracy": 0.6486483410000801,
909
- "num_tokens": 349581218.0,
910
- "step": 900
911
- },
912
- {
913
- "entropy": 1.5319293648004533,
914
- "epoch": 0.43071825819429654,
915
- "grad_norm": 0.16074907779693604,
916
- "learning_rate": 6.543596190570381e-06,
917
- "loss": 1.5406,
918
- "mean_token_accuracy": 0.6477162018418312,
919
- "num_tokens": 353439070.0,
920
- "step": 910
921
- },
922
- {
923
- "entropy": 1.5505404442548751,
924
- "epoch": 0.43545142586676133,
925
- "grad_norm": 0.1633734107017517,
926
- "learning_rate": 6.4689672783550715e-06,
927
- "loss": 1.551,
928
- "mean_token_accuracy": 0.6463420912623405,
929
- "num_tokens": 357338828.0,
930
- "step": 920
931
- },
932
- {
933
- "entropy": 1.5329742044210435,
934
- "epoch": 0.4401845935392261,
935
- "grad_norm": 0.1549346148967743,
936
- "learning_rate": 6.393978444254798e-06,
937
- "loss": 1.5368,
938
- "mean_token_accuracy": 0.6490220680832863,
939
- "num_tokens": 361226544.0,
940
- "step": 930
941
- },
942
- {
943
- "entropy": 1.5339823961257935,
944
- "epoch": 0.4449177612116909,
945
- "grad_norm": 0.15460601449012756,
946
- "learning_rate": 6.3186480618051215e-06,
947
- "loss": 1.5398,
948
- "mean_token_accuracy": 0.6480798035860061,
949
- "num_tokens": 365067789.0,
950
- "step": 940
951
- },
952
- {
953
- "entropy": 1.5356625199317933,
954
- "epoch": 0.4496509288841557,
955
- "grad_norm": 0.1592043936252594,
956
- "learning_rate": 6.242994588226731e-06,
957
- "loss": 1.5378,
958
- "mean_token_accuracy": 0.6489008024334908,
959
- "num_tokens": 368967569.0,
960
- "step": 950
961
- },
962
- {
963
- "entropy": 1.532603034377098,
964
- "epoch": 0.45438409655662054,
965
- "grad_norm": 0.15242810547351837,
966
- "learning_rate": 6.1670365599031215e-06,
967
- "loss": 1.5352,
968
- "mean_token_accuracy": 0.649759191274643,
969
- "num_tokens": 372857431.0,
970
- "step": 960
971
- },
972
- {
973
- "entropy": 1.5232465624809266,
974
- "epoch": 0.45911726422908533,
975
- "grad_norm": 0.1570185422897339,
976
- "learning_rate": 6.090792587838867e-06,
977
- "loss": 1.5277,
978
- "mean_token_accuracy": 0.6510468333959579,
979
- "num_tokens": 376736423.0,
980
- "step": 970
981
- },
982
- {
983
- "entropy": 1.5315758377313613,
984
- "epoch": 0.4638504319015501,
985
- "grad_norm": 0.15817788243293762,
986
- "learning_rate": 6.014281353099601e-06,
987
- "loss": 1.5306,
988
- "mean_token_accuracy": 0.6500695019960403,
989
- "num_tokens": 380611456.0,
990
- "step": 980
991
- },
992
- {
993
- "entropy": 1.5315613538026809,
994
- "epoch": 0.4685835995740149,
995
- "grad_norm": 0.16947001218795776,
996
- "learning_rate": 5.937521602234842e-06,
997
- "loss": 1.5368,
998
- "mean_token_accuracy": 0.6481669098138809,
999
- "num_tokens": 384494701.0,
1000
- "step": 990
1001
- },
1002
- {
1003
- "entropy": 1.541380015015602,
1004
- "epoch": 0.4733167672464797,
1005
- "grad_norm": 0.1462167650461197,
1006
- "learning_rate": 5.8605321426847795e-06,
1007
- "loss": 1.5436,
1008
- "mean_token_accuracy": 0.6473192036151886,
1009
- "num_tokens": 388320020.0,
1010
- "step": 1000
1011
- },
1012
- {
1013
- "entropy": 1.5256280928850174,
1014
- "epoch": 0.4780499349189445,
1015
- "grad_norm": 0.16574805974960327,
1016
- "learning_rate": 5.783331838172116e-06,
1017
- "loss": 1.5349,
1018
- "mean_token_accuracy": 0.6488758757710457,
1019
- "num_tokens": 392189228.0,
1020
- "step": 1010
1021
- },
1022
- {
1023
- "entropy": 1.527175721526146,
1024
- "epoch": 0.4827831025914093,
1025
- "grad_norm": 0.15633495151996613,
1026
- "learning_rate": 5.705939604080147e-06,
1027
- "loss": 1.5316,
1028
- "mean_token_accuracy": 0.6492022335529327,
1029
- "num_tokens": 396051728.0,
1030
- "step": 1020
1031
- },
1032
- {
1033
- "entropy": 1.5351572215557099,
1034
- "epoch": 0.4875162702638741,
1035
- "grad_norm": 0.17186181247234344,
1036
- "learning_rate": 5.628374402818173e-06,
1037
- "loss": 1.54,
1038
- "mean_token_accuracy": 0.6493882149457931,
1039
- "num_tokens": 399969073.0,
1040
- "step": 1030
1041
- },
1042
- {
1043
- "entropy": 1.5383628755807877,
1044
- "epoch": 0.4922494379363389,
1045
- "grad_norm": 0.1601661890745163,
1046
- "learning_rate": 5.550655239175377e-06,
1047
- "loss": 1.5395,
1048
- "mean_token_accuracy": 0.6477264970541,
1049
- "num_tokens": 403812928.0,
1050
- "step": 1040
1051
- },
1052
- {
1053
- "entropy": 1.5366960883140564,
1054
- "epoch": 0.4969826056088037,
1055
- "grad_norm": 0.16334351897239685,
1056
- "learning_rate": 5.472801155664339e-06,
1057
- "loss": 1.539,
1058
- "mean_token_accuracy": 0.6490108296275139,
1059
- "num_tokens": 407675957.0,
1060
- "step": 1050
1061
- },
1062
- {
1063
- "entropy": 1.5218679517507554,
1064
- "epoch": 0.5017157732812685,
1065
- "grad_norm": 0.15565453469753265,
1066
- "learning_rate": 5.394831227855291e-06,
1067
- "loss": 1.5295,
1068
- "mean_token_accuracy": 0.64990915954113,
1069
- "num_tokens": 411549219.0,
1070
- "step": 1060
1071
- },
1072
- {
1073
- "entropy": 1.5312676399946212,
1074
- "epoch": 0.5064489409537333,
1075
- "grad_norm": 0.16980209946632385,
1076
- "learning_rate": 5.3167645597022855e-06,
1077
- "loss": 1.5335,
1078
- "mean_token_accuracy": 0.6499560788273812,
1079
- "num_tokens": 415463466.0,
1080
- "step": 1070
1081
- },
1082
- {
1083
- "entropy": 1.5292291462421417,
1084
- "epoch": 0.5111821086261981,
1085
- "grad_norm": 0.1765119880437851,
1086
- "learning_rate": 5.238620278862397e-06,
1087
- "loss": 1.5346,
1088
- "mean_token_accuracy": 0.649413350224495,
1089
- "num_tokens": 419305289.0,
1090
- "step": 1080
1091
- },
1092
- {
1093
- "entropy": 1.5442948073148728,
1094
- "epoch": 0.5159152762986629,
1095
- "grad_norm": 0.1539527326822281,
1096
- "learning_rate": 5.160417532009122e-06,
1097
- "loss": 1.5398,
1098
- "mean_token_accuracy": 0.6489475637674331,
1099
- "num_tokens": 423200780.0,
1100
- "step": 1090
1101
- },
1102
- {
1103
- "entropy": 1.5311467558145524,
1104
- "epoch": 0.5206484439711276,
1105
- "grad_norm": 0.15670068562030792,
1106
- "learning_rate": 5.082175480141126e-06,
1107
- "loss": 1.5399,
1108
- "mean_token_accuracy": 0.6483099237084389,
1109
- "num_tokens": 427105709.0,
1110
- "step": 1100
1111
- },
1112
- {
1113
- "entropy": 1.5239627480506897,
1114
- "epoch": 0.5253816116435924,
1115
- "grad_norm": 0.15966466069221497,
1116
- "learning_rate": 5.003913293887467e-06,
1117
- "loss": 1.5261,
1118
- "mean_token_accuracy": 0.6510903730988502,
1119
- "num_tokens": 430984481.0,
1120
- "step": 1110
1121
- },
1122
- {
1123
- "entropy": 1.521809309720993,
1124
- "epoch": 0.5301147793160572,
1125
- "grad_norm": 0.15411856770515442,
1126
- "learning_rate": 4.925650148810465e-06,
1127
- "loss": 1.5239,
1128
- "mean_token_accuracy": 0.6515027031302452,
1129
- "num_tokens": 434868955.0,
1130
- "step": 1120
1131
- },
1132
- {
1133
- "entropy": 1.5308598577976227,
1134
- "epoch": 0.5348479469885221,
1135
- "grad_norm": 0.16343414783477783,
1136
- "learning_rate": 4.847405220707378e-06,
1137
- "loss": 1.537,
1138
- "mean_token_accuracy": 0.6491450101137162,
1139
- "num_tokens": 438789538.0,
1140
- "step": 1130
1141
- },
1142
- {
1143
- "entropy": 1.522998222708702,
1144
- "epoch": 0.5395811146609869,
1145
- "grad_norm": 0.1585935652256012,
1146
- "learning_rate": 4.7691976809119835e-06,
1147
- "loss": 1.5227,
1148
- "mean_token_accuracy": 0.6509381130337715,
1149
- "num_tokens": 442659205.0,
1150
- "step": 1140
1151
- },
1152
- {
1153
- "entropy": 1.5199230432510376,
1154
- "epoch": 0.5443142823334517,
1155
- "grad_norm": 0.15242497622966766,
1156
- "learning_rate": 4.691046691597303e-06,
1157
- "loss": 1.5277,
1158
- "mean_token_accuracy": 0.6500678122043609,
1159
- "num_tokens": 446546475.0,
1160
- "step": 1150
1161
- },
1162
- {
1163
- "entropy": 1.5328166007995605,
1164
- "epoch": 0.5490474500059165,
1165
- "grad_norm": 0.1544242948293686,
1166
- "learning_rate": 4.612971401080521e-06,
1167
- "loss": 1.5344,
1168
- "mean_token_accuracy": 0.6496497511863708,
1169
- "num_tokens": 450440495.0,
1170
- "step": 1160
1171
- },
1172
- {
1173
- "entropy": 1.5248778969049455,
1174
- "epoch": 0.5537806176783813,
1175
- "grad_norm": 0.15791839361190796,
1176
- "learning_rate": 4.5349909391313384e-06,
1177
- "loss": 1.529,
1178
- "mean_token_accuracy": 0.6500067859888077,
1179
- "num_tokens": 454368931.0,
1180
- "step": 1170
1181
- },
1182
- {
1183
- "entropy": 1.5288578271865845,
1184
- "epoch": 0.5585137853508461,
1185
- "grad_norm": 0.1565057933330536,
1186
- "learning_rate": 4.457124412284849e-06,
1187
- "loss": 1.5328,
1188
- "mean_token_accuracy": 0.6499333620071411,
1189
- "num_tokens": 458289128.0,
1190
- "step": 1180
1191
- },
1192
- {
1193
- "entropy": 1.5237576812505722,
1194
- "epoch": 0.5632469530233108,
1195
- "grad_norm": 0.16522431373596191,
1196
- "learning_rate": 4.379390899160116e-06,
1197
- "loss": 1.5242,
1198
- "mean_token_accuracy": 0.6513824790716172,
1199
- "num_tokens": 462166520.0,
1200
- "step": 1190
1201
- },
1202
- {
1203
- "entropy": 1.5189936101436614,
1204
- "epoch": 0.5679801206957756,
1205
- "grad_norm": 0.1568940430879593,
1206
- "learning_rate": 4.301809445785582e-06,
1207
- "loss": 1.5221,
1208
- "mean_token_accuracy": 0.6510216951370239,
1209
- "num_tokens": 466060024.0,
1210
- "step": 1200
1211
- },
1212
- {
1213
- "entropy": 1.52001011967659,
1214
- "epoch": 0.5727132883682404,
1215
- "grad_norm": 0.16081492602825165,
1216
- "learning_rate": 4.224399060932477e-06,
1217
- "loss": 1.5253,
1218
- "mean_token_accuracy": 0.6509154736995697,
1219
- "num_tokens": 469988704.0,
1220
- "step": 1210
1221
- },
1222
- {
1223
- "entropy": 1.518327683210373,
1224
- "epoch": 0.5774464560407052,
1225
- "grad_norm": 0.1593891978263855,
1226
- "learning_rate": 4.147178711457343e-06,
1227
- "loss": 1.5268,
1228
- "mean_token_accuracy": 0.6499305665493011,
1229
- "num_tokens": 473843546.0,
1230
- "step": 1220
1231
- },
1232
- {
1233
- "entropy": 1.5157315611839295,
1234
- "epoch": 0.58217962371317,
1235
- "grad_norm": 0.1540728062391281,
1236
- "learning_rate": 4.070167317654829e-06,
1237
- "loss": 1.5215,
1238
- "mean_token_accuracy": 0.6516207203269004,
1239
- "num_tokens": 477705517.0,
1240
- "step": 1230
1241
- },
1242
- {
1243
- "entropy": 1.5173570811748505,
1244
- "epoch": 0.5869127913856348,
1245
- "grad_norm": 0.1539110541343689,
1246
- "learning_rate": 3.9933837486219065e-06,
1247
- "loss": 1.5212,
1248
- "mean_token_accuracy": 0.6517001137137413,
1249
- "num_tokens": 481587746.0,
1250
- "step": 1240
1251
- },
1252
- {
1253
- "entropy": 1.519597727060318,
1254
- "epoch": 0.5916459590580996,
1255
- "grad_norm": 0.1510864943265915,
1256
- "learning_rate": 3.916846817634618e-06,
1257
- "loss": 1.5254,
1258
- "mean_token_accuracy": 0.6502514213323594,
1259
- "num_tokens": 485420622.0,
1260
- "step": 1250
1261
- },
1262
- {
1263
- "entropy": 1.5250845789909362,
1264
- "epoch": 0.5963791267305645,
1265
- "grad_norm": 0.15542279183864594,
1266
- "learning_rate": 3.840575277538495e-06,
1267
- "loss": 1.5268,
1268
- "mean_token_accuracy": 0.6506903722882271,
1269
- "num_tokens": 489322234.0,
1270
- "step": 1260
1271
- },
1272
- {
1273
- "entropy": 1.530037170648575,
1274
- "epoch": 0.6011122944030293,
1275
- "grad_norm": 0.16025404632091522,
1276
- "learning_rate": 3.764587816153813e-06,
1277
- "loss": 1.5328,
1278
- "mean_token_accuracy": 0.6497552871704102,
1279
- "num_tokens": 493213867.0,
1280
- "step": 1270
1281
- },
1282
- {
1283
- "entropy": 1.5264522224664687,
1284
- "epoch": 0.6058454620754941,
1285
- "grad_norm": 0.1563606560230255,
1286
- "learning_rate": 3.688903051696737e-06,
1287
- "loss": 1.5271,
1288
- "mean_token_accuracy": 0.6511508211493492,
1289
- "num_tokens": 497079317.0,
1290
- "step": 1280
1291
- },
1292
- {
1293
- "entropy": 1.517073431611061,
1294
- "epoch": 0.6105786297479588,
1295
- "grad_norm": 0.15244126319885254,
1296
- "learning_rate": 3.61353952821756e-06,
1297
- "loss": 1.519,
1298
- "mean_token_accuracy": 0.6525383740663528,
1299
- "num_tokens": 501003584.0,
1300
- "step": 1290
1301
- },
1302
- {
1303
- "entropy": 1.520869755744934,
1304
- "epoch": 0.6153117974204236,
1305
- "grad_norm": 0.15403762459754944,
1306
- "learning_rate": 3.5385157110570917e-06,
1307
- "loss": 1.5296,
1308
- "mean_token_accuracy": 0.6494565770030022,
1309
- "num_tokens": 504882648.0,
1310
- "step": 1300
1311
- },
1312
- {
1313
- "entropy": 1.5278783291578293,
1314
- "epoch": 0.6200449650928884,
1315
- "grad_norm": 0.1558082103729248,
1316
- "learning_rate": 3.463849982322326e-06,
1317
- "loss": 1.532,
1318
- "mean_token_accuracy": 0.6506448686122894,
1319
- "num_tokens": 508796374.0,
1320
- "step": 1310
1321
- },
1322
- {
1323
- "entropy": 1.5135444432497025,
1324
- "epoch": 0.6247781327653532,
1325
- "grad_norm": 0.16050703823566437,
1326
- "learning_rate": 3.389560636382531e-06,
1327
- "loss": 1.5184,
1328
- "mean_token_accuracy": 0.6528948619961739,
1329
- "num_tokens": 512676763.0,
1330
- "step": 1320
1331
- },
1332
- {
1333
- "entropy": 1.5228564977645873,
1334
- "epoch": 0.629511300437818,
1335
- "grad_norm": 0.1465277224779129,
1336
- "learning_rate": 3.315665875386807e-06,
1337
- "loss": 1.5259,
1338
- "mean_token_accuracy": 0.6503187119960785,
1339
- "num_tokens": 516588257.0,
1340
- "step": 1330
1341
- },
1342
- {
1343
- "entropy": 1.5219411462545396,
1344
- "epoch": 0.6342444681102828,
1345
- "grad_norm": 0.15017631649971008,
1346
- "learning_rate": 3.2421838048042516e-06,
1347
- "loss": 1.5266,
1348
- "mean_token_accuracy": 0.6507244795560837,
1349
- "num_tokens": 520496855.0,
1350
- "step": 1340
1351
- },
1352
- {
1353
- "entropy": 1.5131709426641464,
1354
- "epoch": 0.6389776357827476,
1355
- "grad_norm": 0.1609673649072647,
1356
- "learning_rate": 3.169132428987819e-06,
1357
- "loss": 1.5171,
1358
- "mean_token_accuracy": 0.652211906015873,
1359
- "num_tokens": 524368593.0,
1360
- "step": 1350
1361
- },
1362
- {
1363
- "entropy": 1.5158830434083939,
1364
- "epoch": 0.6437108034552124,
1365
- "grad_norm": 0.1541507989168167,
1366
- "learning_rate": 3.0965296467629413e-06,
1367
- "loss": 1.5207,
1368
- "mean_token_accuracy": 0.6517833650112153,
1369
- "num_tokens": 528292613.0,
1370
- "step": 1360
1371
- },
1372
- {
1373
- "entropy": 1.5173752516508103,
1374
- "epoch": 0.6484439711276772,
1375
- "grad_norm": 0.14874839782714844,
1376
- "learning_rate": 3.024393247042018e-06,
1377
- "loss": 1.5186,
1378
- "mean_token_accuracy": 0.6526758641004562,
1379
- "num_tokens": 532197090.0,
1380
- "step": 1370
1381
- },
1382
- {
1383
- "entropy": 1.5261268466711044,
1384
- "epoch": 0.6531771388001419,
1385
- "grad_norm": 0.1556578129529953,
1386
- "learning_rate": 2.9527409044658286e-06,
1387
- "loss": 1.529,
1388
- "mean_token_accuracy": 0.6497259110212326,
1389
- "num_tokens": 536037282.0,
1390
- "step": 1380
1391
- },
1392
- {
1393
- "entropy": 1.5166011482477189,
1394
- "epoch": 0.6579103064726068,
1395
- "grad_norm": 0.15171046555042267,
1396
- "learning_rate": 2.881590175072948e-06,
1397
- "loss": 1.5189,
1398
- "mean_token_accuracy": 0.6521017536520958,
1399
- "num_tokens": 539892183.0,
1400
- "step": 1390
1401
- },
1402
- {
1403
- "entropy": 1.5153537422418595,
1404
- "epoch": 0.6626434741450716,
1405
- "grad_norm": 0.1611650586128235,
1406
- "learning_rate": 2.8109584919982145e-06,
1407
- "loss": 1.5221,
1408
- "mean_token_accuracy": 0.6510555073618889,
1409
- "num_tokens": 543786829.0,
1410
- "step": 1400
1411
- },
1412
- {
1413
- "entropy": 1.5176646322011949,
1414
- "epoch": 0.6673766418175364,
1415
- "grad_norm": 0.16072982549667358,
1416
- "learning_rate": 2.7408631612013228e-06,
1417
- "loss": 1.5198,
1418
- "mean_token_accuracy": 0.6510667949914932,
1419
- "num_tokens": 547697306.0,
1420
- "step": 1410
1421
- },
1422
- {
1423
- "entropy": 1.516287124156952,
1424
- "epoch": 0.6721098094900012,
1425
- "grad_norm": 0.15958984196186066,
1426
- "learning_rate": 2.671321357226566e-06,
1427
- "loss": 1.5191,
1428
- "mean_token_accuracy": 0.6518496558070183,
1429
- "num_tokens": 551553208.0,
1430
- "step": 1420
1431
- },
1432
- {
1433
- "entropy": 1.5146547496318816,
1434
- "epoch": 0.676842977162466,
1435
- "grad_norm": 0.156447172164917,
1436
- "learning_rate": 2.602350118994782e-06,
1437
- "loss": 1.5178,
1438
- "mean_token_accuracy": 0.6530700176954269,
1439
- "num_tokens": 555450956.0,
1440
- "step": 1430
1441
- },
1442
- {
1443
- "entropy": 1.5136778473854064,
1444
- "epoch": 0.6815761448349308,
1445
- "grad_norm": 0.15450631082057953,
1446
- "learning_rate": 2.5339663456285302e-06,
1447
- "loss": 1.5173,
1448
- "mean_token_accuracy": 0.6531704932451248,
1449
- "num_tokens": 559347478.0,
1450
- "step": 1440
1451
- },
1452
- {
1453
- "entropy": 1.5194635063409805,
1454
- "epoch": 0.6863093125073956,
1455
- "grad_norm": 0.1552649736404419,
1456
- "learning_rate": 2.466186792311522e-06,
1457
- "loss": 1.5282,
1458
- "mean_token_accuracy": 0.6514062762260437,
1459
- "num_tokens": 563180332.0,
1460
- "step": 1450
1461
- },
1462
- {
1463
- "entropy": 1.5103699028491975,
1464
- "epoch": 0.6910424801798604,
1465
- "grad_norm": 0.15950508415699005,
1466
- "learning_rate": 2.399028066183306e-06,
1467
- "loss": 1.515,
1468
- "mean_token_accuracy": 0.6529736325144768,
1469
- "num_tokens": 567111777.0,
1470
- "step": 1460
1471
- },
1472
- {
1473
- "entropy": 1.515543583035469,
1474
- "epoch": 0.6957756478523252,
1475
- "grad_norm": 0.1573014110326767,
1476
- "learning_rate": 2.332506622270256e-06,
1477
- "loss": 1.5143,
1478
- "mean_token_accuracy": 0.6537506148219109,
1479
- "num_tokens": 570991505.0,
1480
- "step": 1470
1481
- },
1482
- {
1483
- "entropy": 1.5154441595077515,
1484
- "epoch": 0.7005088155247899,
1485
- "grad_norm": 0.1603464037179947,
1486
- "learning_rate": 2.2666387594537896e-06,
1487
- "loss": 1.5204,
1488
- "mean_token_accuracy": 0.652016893029213,
1489
- "num_tokens": 574883883.0,
1490
- "step": 1480
1491
- },
1492
- {
1493
- "entropy": 1.5132777452468873,
1494
- "epoch": 0.7052419831972547,
1495
- "grad_norm": 0.15662123262882233,
1496
- "learning_rate": 2.201440616476885e-06,
1497
- "loss": 1.5177,
1498
- "mean_token_accuracy": 0.6527584567666054,
1499
- "num_tokens": 578724891.0,
1500
- "step": 1490
1501
- },
1502
- {
1503
- "entropy": 1.5139577269554139,
1504
- "epoch": 0.7099751508697195,
1505
- "grad_norm": 0.15398600697517395,
1506
- "learning_rate": 2.136928167989803e-06,
1507
- "loss": 1.5188,
1508
- "mean_token_accuracy": 0.6523755803704262,
1509
- "num_tokens": 582630649.0,
1510
- "step": 1500
1511
- },
1512
- {
1513
- "entropy": 1.5097831279039382,
1514
- "epoch": 0.7147083185421843,
1515
- "grad_norm": 0.1604987233877182,
1516
- "learning_rate": 2.073117220636027e-06,
1517
- "loss": 1.517,
1518
- "mean_token_accuracy": 0.653074149787426,
1519
- "num_tokens": 586509280.0,
1520
- "step": 1510
1521
- },
1522
- {
1523
- "entropy": 1.5180349677801133,
1524
- "epoch": 0.7194414862146492,
1525
- "grad_norm": 0.15329141914844513,
1526
- "learning_rate": 2.0100234091793778e-06,
1527
- "loss": 1.5201,
1528
- "mean_token_accuracy": 0.6520388692617416,
1529
- "num_tokens": 590387510.0,
1530
- "step": 1520
1531
- },
1532
- {
1533
- "entropy": 1.5136069297790526,
1534
- "epoch": 0.724174653887114,
1535
- "grad_norm": 0.15793868899345398,
1536
- "learning_rate": 1.9476621926732227e-06,
1537
- "loss": 1.5221,
1538
- "mean_token_accuracy": 0.6520983532071114,
1539
- "num_tokens": 594314494.0,
1540
- "step": 1530
1541
- },
1542
- {
1543
- "entropy": 1.5157446891069413,
1544
- "epoch": 0.7289078215595788,
1545
- "grad_norm": 0.15694737434387207,
1546
- "learning_rate": 1.8860488506727409e-06,
1547
- "loss": 1.5175,
1548
- "mean_token_accuracy": 0.6533299177885056,
1549
- "num_tokens": 598258201.0,
1550
- "step": 1540
1551
- },
1552
- {
1553
- "entropy": 1.522775200009346,
1554
- "epoch": 0.7336409892320436,
1555
- "grad_norm": 0.15210026502609253,
1556
- "learning_rate": 1.8251984794911887e-06,
1557
- "loss": 1.526,
1558
- "mean_token_accuracy": 0.6508068576455116,
1559
- "num_tokens": 602110882.0,
1560
- "step": 1550
1561
- },
1562
- {
1563
- "entropy": 1.5086775600910187,
1564
- "epoch": 0.7383741569045084,
1565
- "grad_norm": 0.15608255565166473,
1566
- "learning_rate": 1.765125988501034e-06,
1567
- "loss": 1.513,
1568
- "mean_token_accuracy": 0.6531857639551163,
1569
- "num_tokens": 606013390.0,
1570
- "step": 1560
1571
- },
1572
- {
1573
- "entropy": 1.5057473361492157,
1574
- "epoch": 0.7431073245769731,
1575
- "grad_norm": 0.1536129117012024,
1576
- "learning_rate": 1.7058460964809086e-06,
1577
- "loss": 1.5105,
1578
- "mean_token_accuracy": 0.6533811703324318,
1579
- "num_tokens": 609866962.0,
1580
- "step": 1570
1581
- },
1582
- {
1583
- "entropy": 1.5125353664159775,
1584
- "epoch": 0.7478404922494379,
1585
- "grad_norm": 0.15434998273849487,
1586
- "learning_rate": 1.6473733280092658e-06,
1587
- "loss": 1.5163,
1588
- "mean_token_accuracy": 0.6529958829283714,
1589
- "num_tokens": 613748468.0,
1590
- "step": 1580
1591
- },
1592
- {
1593
- "entropy": 1.5094835668802262,
1594
- "epoch": 0.7525736599219027,
1595
- "grad_norm": 0.16267992556095123,
1596
- "learning_rate": 1.589722009905606e-06,
1597
- "loss": 1.512,
1598
- "mean_token_accuracy": 0.6539744392037392,
1599
- "num_tokens": 617586252.0,
1600
- "step": 1590
1601
- },
1602
- {
1603
- "entropy": 1.5102200895547866,
1604
- "epoch": 0.7573068275943675,
1605
- "grad_norm": 0.15085026621818542,
1606
- "learning_rate": 1.5329062677201594e-06,
1607
- "loss": 1.5152,
1608
- "mean_token_accuracy": 0.652720046043396,
1609
- "num_tokens": 621473159.0,
1610
- "step": 1600
1611
- },
1612
- {
1613
- "entropy": 1.5119531810283662,
1614
- "epoch": 0.7620399952668323,
1615
- "grad_norm": 0.15508662164211273,
1616
- "learning_rate": 1.4769400222728974e-06,
1617
- "loss": 1.5176,
1618
- "mean_token_accuracy": 0.6533014222979545,
1619
- "num_tokens": 625366303.0,
1620
- "step": 1610
1621
- },
1622
- {
1623
- "entropy": 1.5167303442955018,
1624
- "epoch": 0.7667731629392971,
1625
- "grad_norm": 0.14820463955402374,
1626
- "learning_rate": 1.4218369862426896e-06,
1627
- "loss": 1.5186,
1628
- "mean_token_accuracy": 0.6529193088412285,
1629
- "num_tokens": 629214314.0,
1630
- "step": 1620
1631
- },
1632
- {
1633
- "entropy": 1.510753783583641,
1634
- "epoch": 0.7715063306117619,
1635
- "grad_norm": 0.1586351990699768,
1636
- "learning_rate": 1.3676106608074602e-06,
1637
- "loss": 1.5144,
1638
- "mean_token_accuracy": 0.653745847940445,
1639
- "num_tokens": 633129219.0,
1640
- "step": 1630
1641
- },
1642
- {
1643
- "entropy": 1.5246556103229523,
1644
- "epoch": 0.7762394982842267,
1645
- "grad_norm": 0.15720601379871368,
1646
- "learning_rate": 1.3142743323361818e-06,
1647
- "loss": 1.5293,
1648
- "mean_token_accuracy": 0.6503741011023522,
1649
- "num_tokens": 636956832.0,
1650
- "step": 1640
1651
- },
1652
- {
1653
- "entropy": 1.518925702571869,
1654
- "epoch": 0.7809726659566916,
1655
- "grad_norm": 0.15746639668941498,
1656
- "learning_rate": 1.2618410691334805e-06,
1657
- "loss": 1.5244,
1658
- "mean_token_accuracy": 0.6520501941442489,
1659
- "num_tokens": 640891051.0,
1660
- "step": 1650
1661
- },
1662
- {
1663
- "entropy": 1.51315179169178,
1664
- "epoch": 0.7857058336291564,
1665
- "grad_norm": 0.15106885135173798,
1666
- "learning_rate": 1.2103237182376848e-06,
1667
- "loss": 1.5184,
1668
- "mean_token_accuracy": 0.6530534133315087,
1669
- "num_tokens": 644765813.0,
1670
- "step": 1660
1671
- },
1672
- {
1673
- "entropy": 1.5101027220487595,
1674
- "epoch": 0.7904390013016211,
1675
- "grad_norm": 0.15605495870113373,
1676
- "learning_rate": 1.159734902273078e-06,
1677
- "loss": 1.5137,
1678
- "mean_token_accuracy": 0.6541712284088135,
1679
- "num_tokens": 648685831.0,
1680
- "step": 1670
1681
- },
1682
- {
1683
- "entropy": 1.5037204265594482,
1684
- "epoch": 0.7951721689740859,
1685
- "grad_norm": 0.15608017146587372,
1686
- "learning_rate": 1.1100870163571486e-06,
1687
- "loss": 1.5085,
1688
- "mean_token_accuracy": 0.6543751254677772,
1689
- "num_tokens": 652549548.0,
1690
- "step": 1680
1691
- },
1692
- {
1693
- "entropy": 1.5166430234909059,
1694
- "epoch": 0.7999053366465507,
1695
- "grad_norm": 0.15508434176445007,
1696
- "learning_rate": 1.061392225063571e-06,
1697
- "loss": 1.524,
1698
- "mean_token_accuracy": 0.6517237603664399,
1699
- "num_tokens": 656455494.0,
1700
- "step": 1690
1701
- },
1702
- {
1703
- "entropy": 1.5094672977924346,
1704
- "epoch": 0.8046385043190155,
1705
- "grad_norm": 0.1599401831626892,
1706
- "learning_rate": 1.0136624594416828e-06,
1707
- "loss": 1.5145,
1708
- "mean_token_accuracy": 0.6521976351737976,
1709
- "num_tokens": 660321875.0,
1710
- "step": 1700
1711
- },
1712
- {
1713
- "entropy": 1.524332845211029,
1714
- "epoch": 0.8093716719914803,
1715
- "grad_norm": 0.15658311545848846,
1716
- "learning_rate": 9.669094140931678e-07,
1717
- "loss": 1.5249,
1718
- "mean_token_accuracy": 0.651316037774086,
1719
- "num_tokens": 664173208.0,
1720
- "step": 1710
1721
- },
1722
- {
1723
- "entropy": 1.5061070144176483,
1724
- "epoch": 0.8141048396639451,
1725
- "grad_norm": 0.15877439081668854,
1726
- "learning_rate": 9.211445443066896e-07,
1727
- "loss": 1.5095,
1728
- "mean_token_accuracy": 0.6547744259238243,
1729
- "num_tokens": 668090711.0,
1730
- "step": 1720
1731
- },
1732
- {
1733
- "entropy": 1.5110326170921327,
1734
- "epoch": 0.8188380073364099,
1735
- "grad_norm": 0.15988118946552277,
1736
- "learning_rate": 8.763790632511482e-07,
1737
- "loss": 1.52,
1738
- "mean_token_accuracy": 0.6521027386188507,
1739
- "num_tokens": 671930932.0,
1740
- "step": 1730
1741
- },
1742
- {
1743
- "entropy": 1.5050281405448913,
1744
- "epoch": 0.8235711750088747,
1745
- "grad_norm": 0.15555670857429504,
1746
- "learning_rate": 8.326239392282586e-07,
1747
- "loss": 1.5105,
1748
- "mean_token_accuracy": 0.6531902849674225,
1749
- "num_tokens": 675792125.0,
1750
- "step": 1740
1751
- },
1752
- {
1753
- "entropy": 1.5129504561424256,
1754
- "epoch": 0.8283043426813395,
1755
- "grad_norm": 0.16341622173786163,
1756
- "learning_rate": 7.898898929851406e-07,
1757
- "loss": 1.5162,
1758
- "mean_token_accuracy": 0.6542815431952477,
1759
- "num_tokens": 679648075.0,
1760
- "step": 1750
1761
- },
1762
- {
1763
- "entropy": 1.5132198542356492,
1764
- "epoch": 0.8330375103538042,
1765
- "grad_norm": 0.1575479656457901,
1766
- "learning_rate": 7.481873950875468e-07,
1767
- "loss": 1.5167,
1768
- "mean_token_accuracy": 0.6529857292771339,
1769
- "num_tokens": 683496935.0,
1770
- "step": 1760
1771
- },
1772
- {
1773
- "entropy": 1.514217236638069,
1774
- "epoch": 0.837770678026269,
1775
- "grad_norm": 0.16133052110671997,
1776
- "learning_rate": 7.075266633543959e-07,
1777
- "loss": 1.5176,
1778
- "mean_token_accuracy": 0.6526405304670334,
1779
- "num_tokens": 687412196.0,
1780
- "step": 1770
1781
- },
1782
- {
1783
- "entropy": 1.5245876908302307,
1784
- "epoch": 0.8425038456987339,
1785
- "grad_norm": 0.1658501774072647,
1786
- "learning_rate": 6.679176603542398e-07,
1787
- "loss": 1.5267,
1788
- "mean_token_accuracy": 0.6507634833455086,
1789
- "num_tokens": 691261759.0,
1790
- "step": 1780
1791
- },
1792
- {
1793
- "entropy": 1.520870679616928,
1794
- "epoch": 0.8472370133711987,
1795
- "grad_norm": 0.15542222559452057,
1796
- "learning_rate": 6.29370090964262e-07,
1797
- "loss": 1.5237,
1798
- "mean_token_accuracy": 0.6512942254543305,
1799
- "num_tokens": 695146169.0,
1800
- "step": 1790
1801
- },
1802
- {
1803
- "entropy": 1.5049445867538451,
1804
- "epoch": 0.8519701810436635,
1805
- "grad_norm": 0.15088842809200287,
1806
- "learning_rate": 5.918933999924143e-07,
1807
- "loss": 1.507,
1808
- "mean_token_accuracy": 0.6546556517481804,
1809
- "num_tokens": 699030254.0,
1810
- "step": 1800
1811
- },
1812
- {
1813
- "entropy": 1.5152593225240707,
1814
- "epoch": 0.8567033487161283,
1815
- "grad_norm": 0.15672406554222107,
1816
- "learning_rate": 5.55496769863288e-07,
1817
- "loss": 1.5197,
1818
- "mean_token_accuracy": 0.652678707242012,
1819
- "num_tokens": 702916640.0,
1820
- "step": 1810
1821
- },
1822
- {
1823
- "entropy": 1.5112317591905593,
1824
- "epoch": 0.8614365163885931,
1825
- "grad_norm": 0.15444321930408478,
1826
- "learning_rate": 5.201891183682545e-07,
1827
- "loss": 1.5167,
1828
- "mean_token_accuracy": 0.6519044652581215,
1829
- "num_tokens": 706729238.0,
1830
- "step": 1820
1831
- },
1832
- {
1833
- "entropy": 1.5071990102529527,
1834
- "epoch": 0.8661696840610579,
1835
- "grad_norm": 0.1569969356060028,
1836
- "learning_rate": 4.859790964804617e-07,
1837
- "loss": 1.5104,
1838
- "mean_token_accuracy": 0.6534604758024216,
1839
- "num_tokens": 710608784.0,
1840
- "step": 1830
1841
- },
1842
- {
1843
- "entropy": 1.5036645025014876,
1844
- "epoch": 0.8709028517335227,
1845
- "grad_norm": 0.16283676028251648,
1846
- "learning_rate": 4.5287508623519773e-07,
1847
- "loss": 1.5068,
1848
- "mean_token_accuracy": 0.6550710022449493,
1849
- "num_tokens": 714514707.0,
1850
- "step": 1840
1851
- },
1852
- {
1853
- "entropy": 1.5189965814352036,
1854
- "epoch": 0.8756360194059875,
1855
- "grad_norm": 0.15639755129814148,
1856
- "learning_rate": 4.2088519867614585e-07,
1857
- "loss": 1.5222,
1858
- "mean_token_accuracy": 0.6517624288797379,
1859
- "num_tokens": 718366975.0,
1860
- "step": 1850
1861
- },
1862
- {
1863
- "entropy": 1.5078352093696594,
1864
- "epoch": 0.8803691870784522,
1865
- "grad_norm": 0.1577681601047516,
1866
- "learning_rate": 3.900172718680484e-07,
1867
- "loss": 1.5131,
1868
- "mean_token_accuracy": 0.6530882626771927,
1869
- "num_tokens": 722251285.0,
1870
- "step": 1860
1871
- },
1872
- {
1873
- "entropy": 1.5013979017734527,
1874
- "epoch": 0.885102354750917,
1875
- "grad_norm": 0.16839657723903656,
1876
- "learning_rate": 3.6027886897624376e-07,
1877
- "loss": 1.5079,
1878
- "mean_token_accuracy": 0.6545991912484169,
1879
- "num_tokens": 726162080.0,
1880
- "step": 1870
1881
- },
1882
- {
1883
- "entropy": 1.5258357018232345,
1884
- "epoch": 0.8898355224233818,
1885
- "grad_norm": 0.1610095053911209,
1886
- "learning_rate": 3.3167727641356064e-07,
1887
- "loss": 1.529,
1888
- "mean_token_accuracy": 0.6494020417332649,
1889
- "num_tokens": 730048788.0,
1890
- "step": 1880
1891
- },
1892
- {
1893
- "entropy": 1.5098802983760833,
1894
- "epoch": 0.8945686900958466,
1895
- "grad_norm": 0.15612637996673584,
1896
- "learning_rate": 3.0421950205502813e-07,
1897
- "loss": 1.5136,
1898
- "mean_token_accuracy": 0.6532332092523575,
1899
- "num_tokens": 733943282.0,
1900
- "step": 1890
1901
- },
1902
- {
1903
- "entropy": 1.5100417882204056,
1904
- "epoch": 0.8993018577683114,
1905
- "grad_norm": 0.154027059674263,
1906
- "learning_rate": 2.779122735208267e-07,
1907
- "loss": 1.5139,
1908
- "mean_token_accuracy": 0.6538966566324234,
1909
- "num_tokens": 737803762.0,
1910
- "step": 1900
1911
- },
1912
- {
1913
- "entropy": 1.511214417219162,
1914
- "epoch": 0.9040350254407762,
1915
- "grad_norm": 0.1588786244392395,
1916
- "learning_rate": 2.527620365279021e-07,
1917
- "loss": 1.5141,
1918
- "mean_token_accuracy": 0.6533711299300193,
1919
- "num_tokens": 741703649.0,
1920
- "step": 1910
1921
- },
1922
- {
1923
- "entropy": 1.5116032123565675,
1924
- "epoch": 0.9087681931132411,
1925
- "grad_norm": 0.15595540404319763,
1926
- "learning_rate": 2.2877495331066312e-07,
1927
- "loss": 1.5156,
1928
- "mean_token_accuracy": 0.6534523665904999,
1929
- "num_tokens": 745612891.0,
1930
- "step": 1920
1931
- },
1932
- {
1933
- "entropy": 1.5098071992397308,
1934
- "epoch": 0.9135013607857059,
1935
- "grad_norm": 0.160483717918396,
1936
- "learning_rate": 2.0595690111112398e-07,
1937
- "loss": 1.5142,
1938
- "mean_token_accuracy": 0.6534514829516411,
1939
- "num_tokens": 749494675.0,
1940
- "step": 1930
1941
- },
1942
- {
1943
- "entropy": 1.5071230858564377,
1944
- "epoch": 0.9182345284581707,
1945
- "grad_norm": 0.15629717707633972,
1946
- "learning_rate": 1.8431347073888217e-07,
1947
- "loss": 1.5111,
1948
- "mean_token_accuracy": 0.6530539333820343,
1949
- "num_tokens": 753341723.0,
1950
- "step": 1940
1951
- },
1952
- {
1953
- "entropy": 1.5019009202718734,
1954
- "epoch": 0.9229676961306355,
1955
- "grad_norm": 0.15971773862838745,
1956
- "learning_rate": 1.638499652012754e-07,
1957
- "loss": 1.5059,
1958
- "mean_token_accuracy": 0.6544443607330322,
1959
- "num_tokens": 757233014.0,
1960
- "step": 1950
1961
- },
1962
- {
1963
- "entropy": 1.5118261843919754,
1964
- "epoch": 0.9277008638031002,
1965
- "grad_norm": 0.15779262781143188,
1966
- "learning_rate": 1.44571398404057e-07,
1967
- "loss": 1.515,
1968
- "mean_token_accuracy": 0.6532472312450409,
1969
- "num_tokens": 761115255.0,
1970
- "step": 1960
1971
- },
1972
- {
1973
- "entropy": 1.5036541551351548,
1974
- "epoch": 0.932434031475565,
1975
- "grad_norm": 0.1535780280828476,
1976
- "learning_rate": 1.2648249392289925e-07,
1977
- "loss": 1.5061,
1978
- "mean_token_accuracy": 0.6545781284570694,
1979
- "num_tokens": 765028626.0,
1980
- "step": 1970
1981
- },
1982
- {
1983
- "entropy": 1.50391583442688,
1984
- "epoch": 0.9371671991480298,
1985
- "grad_norm": 0.15700839459896088,
1986
- "learning_rate": 1.095876838460408e-07,
1987
- "loss": 1.506,
1988
- "mean_token_accuracy": 0.6557769432663918,
1989
- "num_tokens": 768939705.0,
1990
- "step": 1980
1991
- },
1992
- {
1993
- "entropy": 1.5056363612413406,
1994
- "epoch": 0.9419003668204946,
1995
- "grad_norm": 0.15855862200260162,
1996
- "learning_rate": 9.38911076883503e-08,
1997
- "loss": 1.5082,
1998
- "mean_token_accuracy": 0.6543473988771439,
1999
- "num_tokens": 772833689.0,
2000
- "step": 1990
2001
- },
2002
- {
2003
- "entropy": 1.525366762280464,
2004
- "epoch": 0.9466335344929594,
2005
- "grad_norm": 0.15759994089603424,
2006
- "learning_rate": 7.939661137707167e-08,
2007
- "loss": 1.5292,
2008
- "mean_token_accuracy": 0.650723172724247,
2009
- "num_tokens": 776737000.0,
2010
- "step": 2000
2011
- },
2012
- {
2013
- "entropy": 1.5107537627220153,
2014
- "epoch": 0.9513667021654242,
2015
- "grad_norm": 0.14716793596744537,
2016
- "learning_rate": 6.610774630951167e-08,
2017
- "loss": 1.514,
2018
- "mean_token_accuracy": 0.6538170203566551,
2019
- "num_tokens": 780623496.0,
2020
- "step": 2010
2021
- },
2022
- {
2023
- "entropy": 1.510766288638115,
2024
- "epoch": 0.956099869837889,
2025
- "grad_norm": 0.15204967558383942,
2026
- "learning_rate": 5.402776848288216e-08,
2027
- "loss": 1.5127,
2028
- "mean_token_accuracy": 0.6531319335103035,
2029
- "num_tokens": 784511197.0,
2030
- "step": 2020
2031
- },
2032
- {
2033
- "entropy": 1.5045825749635697,
2034
- "epoch": 0.9608330375103538,
2035
- "grad_norm": 0.162211611866951,
2036
- "learning_rate": 4.315963769652931e-08,
2037
- "loss": 1.5063,
2038
- "mean_token_accuracy": 0.6537961974740029,
2039
- "num_tokens": 788407251.0,
2040
- "step": 2030
2041
- },
2042
- {
2043
- "entropy": 1.5063252121210098,
2044
- "epoch": 0.9655662051828185,
2045
- "grad_norm": 0.15293368697166443,
2046
- "learning_rate": 3.350601682673094e-08,
2047
- "loss": 1.5086,
2048
- "mean_token_accuracy": 0.654248197376728,
2049
- "num_tokens": 792249083.0,
2050
- "step": 2040
2051
- },
2052
- {
2053
- "entropy": 1.5172866135835648,
2054
- "epoch": 0.9702993728552834,
2055
- "grad_norm": 0.15927255153656006,
2056
- "learning_rate": 2.5069271174247844e-08,
2057
- "loss": 1.5187,
2058
- "mean_token_accuracy": 0.6526034936308861,
2059
- "num_tokens": 796138376.0,
2060
- "step": 2050
2061
- },
2062
- {
2063
- "entropy": 1.515240240097046,
2064
- "epoch": 0.9750325405277482,
2065
- "grad_norm": 0.15734337270259857,
2066
- "learning_rate": 1.785146788478298e-08,
2067
- "loss": 1.5168,
2068
- "mean_token_accuracy": 0.6522197261452675,
2069
- "num_tokens": 800048611.0,
2070
- "step": 2060
2071
- },
2072
- {
2073
- "entropy": 1.505976414680481,
2074
- "epoch": 0.979765708200213,
2075
- "grad_norm": 0.15182293951511383,
2076
- "learning_rate": 1.1854375442498234e-08,
2077
- "loss": 1.5083,
2078
- "mean_token_accuracy": 0.653746984899044,
2079
- "num_tokens": 803953901.0,
2080
- "step": 2070
2081
- },
2082
- {
2083
- "entropy": 1.5067311346530914,
2084
- "epoch": 0.9844988758726778,
2085
- "grad_norm": 0.15344145894050598,
2086
- "learning_rate": 7.0794632367032765e-09,
2087
- "loss": 1.5107,
2088
- "mean_token_accuracy": 0.6533343732357025,
2089
- "num_tokens": 807795934.0,
2090
- "step": 2080
2091
- },
2092
- {
2093
- "entropy": 1.5039544194936751,
2094
- "epoch": 0.9892320435451426,
2095
- "grad_norm": 0.15625813603401184,
2096
- "learning_rate": 3.5279012018324397e-09,
2097
- "loss": 1.5083,
2098
- "mean_token_accuracy": 0.6547796651721001,
2099
- "num_tokens": 811672777.0,
2100
- "step": 2090
2101
- },
2102
- {
2103
- "entropy": 1.5102417439222335,
2104
- "epoch": 0.9939652112176074,
2105
- "grad_norm": 0.14735041558742523,
2106
- "learning_rate": 1.2005595307906792e-09,
2107
- "loss": 1.5121,
2108
- "mean_token_accuracy": 0.6533789873123169,
2109
- "num_tokens": 815540486.0,
2110
- "step": 2100
2111
- },
2112
- {
2113
- "entropy": 1.5077374666929244,
2114
- "epoch": 0.9986983788900722,
2115
- "grad_norm": 0.15531828999519348,
2116
- "learning_rate": 9.800846173968481e-11,
2117
- "loss": 1.5117,
2118
- "mean_token_accuracy": 0.6538635641336441,
2119
- "num_tokens": 819362988.0,
2120
- "step": 2110
2121
- }
2122
- ],
2123
- "logging_steps": 10,
2124
- "max_steps": 2113,
2125
- "num_input_tokens_seen": 0,
2126
- "num_train_epochs": 1,
2127
- "save_steps": 100,
2128
- "stateful_callbacks": {
2129
- "TrainerControl": {
2130
- "args": {
2131
- "should_epoch_stop": false,
2132
- "should_evaluate": false,
2133
- "should_log": false,
2134
- "should_save": true,
2135
- "should_training_stop": true
2136
- },
2137
- "attributes": {}
2138
- }
2139
- },
2140
- "total_flos": 3.6414323458230452e+19,
2141
- "train_batch_size": 4,
2142
- "trial_name": null,
2143
- "trial_params": null
2144
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:57ae05cf930d7a2b45036327e0bf0a2b89948bc10b3b72ad80403976f79712bf
3
- size 7569
 
 
 
 
checkpoint-2113/video_preprocessor_config.json DELETED
@@ -1,41 +0,0 @@
1
- {
2
- "crop_size": null,
3
- "data_format": "channels_first",
4
- "default_to_square": true,
5
- "device": null,
6
- "do_center_crop": null,
7
- "do_convert_rgb": true,
8
- "do_normalize": true,
9
- "do_rescale": true,
10
- "do_resize": true,
11
- "do_sample_frames": true,
12
- "fps": 2,
13
- "image_mean": [
14
- 0.5,
15
- 0.5,
16
- 0.5
17
- ],
18
- "image_std": [
19
- 0.5,
20
- 0.5,
21
- 0.5
22
- ],
23
- "input_data_format": null,
24
- "max_frames": 768,
25
- "merge_size": 2,
26
- "min_frames": 4,
27
- "num_frames": null,
28
- "pad_size": null,
29
- "patch_size": 16,
30
- "processor_class": "Qwen3VLProcessor",
31
- "resample": 3,
32
- "rescale_factor": 0.00392156862745098,
33
- "return_metadata": false,
34
- "size": {
35
- "longest_edge": 25165824,
36
- "shortest_edge": 4096
37
- },
38
- "temporal_patch_size": 2,
39
- "video_metadata": null,
40
- "video_processor_type": "Qwen3VLVideoProcessor"
41
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2113/vocab.json DELETED
The diff for this file is too large to render. See raw diff
 
checkpoint-2113/zero_to_fp32.py DELETED
@@ -1,760 +0,0 @@
1
- #!/usr/bin/env python
2
-
3
- # Copyright (c) Microsoft Corporation.
4
- # SPDX-License-Identifier: Apache-2.0
5
-
6
- # DeepSpeed Team
7
-
8
- # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
- # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
- # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
- # application.
12
- #
13
- # example:
14
- # python zero_to_fp32.py . output_dir/
15
- # or
16
- # python zero_to_fp32.py . output_dir/ --safe_serialization
17
-
18
- import argparse
19
- import torch
20
- import glob
21
- import math
22
- import os
23
- import re
24
- import gc
25
- import json
26
- import numpy as np
27
- from tqdm import tqdm
28
- from collections import OrderedDict
29
- from dataclasses import dataclass
30
-
31
- # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
- # DeepSpeed data structures it has to be available in the current python environment.
33
- from deepspeed.utils import logger
34
- from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
- FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
- FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
-
38
-
39
- @dataclass
40
- class zero_model_state:
41
- buffers: dict()
42
- param_shapes: dict()
43
- shared_params: list
44
- ds_version: int
45
- frozen_param_shapes: dict()
46
- frozen_param_fragments: dict()
47
-
48
-
49
- debug = 0
50
-
51
- # load to cpu
52
- device = torch.device('cpu')
53
-
54
-
55
- def atoi(text):
56
- return int(text) if text.isdigit() else text
57
-
58
-
59
- def natural_keys(text):
60
- '''
61
- alist.sort(key=natural_keys) sorts in human order
62
- http://nedbatchelder.com/blog/200712/human_sorting.html
63
- (See Toothy's implementation in the comments)
64
- '''
65
- return [atoi(c) for c in re.split(r'(\d+)', text)]
66
-
67
-
68
- def get_model_state_file(checkpoint_dir, zero_stage):
69
- if not os.path.isdir(checkpoint_dir):
70
- raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
-
72
- # there should be only one file
73
- if zero_stage <= 2:
74
- file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
- elif zero_stage == 3:
76
- file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
-
78
- if not os.path.exists(file):
79
- raise FileNotFoundError(f"can't find model states file at '{file}'")
80
-
81
- return file
82
-
83
-
84
- def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
- # XXX: need to test that this simple glob rule works for multi-node setup too
86
- ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
-
88
- if len(ckpt_files) == 0:
89
- raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
-
91
- return ckpt_files
92
-
93
-
94
- def get_optim_files(checkpoint_dir):
95
- return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
-
97
-
98
- def get_model_state_files(checkpoint_dir):
99
- return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
-
101
-
102
- def parse_model_states(files):
103
- zero_model_states = []
104
- for file in files:
105
- state_dict = torch.load(file, map_location=device, weights_only=False)
106
-
107
- if BUFFER_NAMES not in state_dict:
108
- raise ValueError(f"{file} is not a model state checkpoint")
109
- buffer_names = state_dict[BUFFER_NAMES]
110
- if debug:
111
- print("Found buffers:", buffer_names)
112
-
113
- # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
- buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
- param_shapes = state_dict[PARAM_SHAPES]
116
-
117
- # collect parameters that are included in param_shapes
118
- param_names = []
119
- for s in param_shapes:
120
- for name in s.keys():
121
- param_names.append(name)
122
-
123
- # update with frozen parameters
124
- frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
- if frozen_param_shapes is not None:
126
- if debug:
127
- print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
- param_names += list(frozen_param_shapes.keys())
129
-
130
- # handle shared params
131
- shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
-
133
- ds_version = state_dict.get(DS_VERSION, None)
134
-
135
- frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
-
137
- z_model_state = zero_model_state(buffers=buffers,
138
- param_shapes=param_shapes,
139
- shared_params=shared_params,
140
- ds_version=ds_version,
141
- frozen_param_shapes=frozen_param_shapes,
142
- frozen_param_fragments=frozen_param_fragments)
143
- zero_model_states.append(z_model_state)
144
-
145
- return zero_model_states
146
-
147
-
148
- def parse_optim_states(files, ds_checkpoint_dir):
149
- total_files = len(files)
150
- state_dicts = []
151
- for f in tqdm(files, desc='Loading checkpoint shards'):
152
- state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
- # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
- # and also handle the case where it was already removed by another helper script
155
- state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
- state_dicts.append(state_dict)
157
-
158
- if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
- raise ValueError(f"{files[0]} is not a zero checkpoint")
160
- zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
- world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
-
163
- # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
- # parameters can be different from data parallelism for non-expert parameters. So we can just
165
- # use the max of the partition_count to get the dp world_size.
166
-
167
- if type(world_size) is list:
168
- world_size = max(world_size)
169
-
170
- if world_size != total_files:
171
- raise ValueError(
172
- f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
- "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
- )
175
-
176
- # the groups are named differently in each stage
177
- if zero_stage <= 2:
178
- fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
- elif zero_stage == 3:
180
- fp32_groups_key = FP32_FLAT_GROUPS
181
- else:
182
- raise ValueError(f"unknown zero stage {zero_stage}")
183
-
184
- fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
- return zero_stage, world_size, fp32_flat_groups
186
-
187
-
188
- def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
- """
190
- Returns fp32 state_dict reconstructed from ds checkpoint
191
-
192
- Args:
193
- - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
-
195
- """
196
- print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
-
198
- optim_files = get_optim_files(ds_checkpoint_dir)
199
- zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
- print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
-
202
- model_files = get_model_state_files(ds_checkpoint_dir)
203
-
204
- zero_model_states = parse_model_states(model_files)
205
- print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
-
207
- if zero_stage <= 2:
208
- return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
- exclude_frozen_parameters)
210
- elif zero_stage == 3:
211
- return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
- exclude_frozen_parameters)
213
-
214
-
215
- def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
- if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
- return
218
-
219
- frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
- frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
-
222
- if debug:
223
- num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
- print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
-
226
- wanted_params = len(frozen_param_shapes)
227
- wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
- avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
- print(f'Frozen params: Have {avail_numel} numels to process.')
230
- print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
-
232
- total_params = 0
233
- total_numel = 0
234
- for name, shape in frozen_param_shapes.items():
235
- total_params += 1
236
- unpartitioned_numel = shape.numel()
237
- total_numel += unpartitioned_numel
238
-
239
- state_dict[name] = frozen_param_fragments[name]
240
-
241
- if debug:
242
- print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
-
244
- print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
-
246
-
247
- def _has_callable(obj, fn):
248
- attr = getattr(obj, fn, None)
249
- return callable(attr)
250
-
251
-
252
- def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
- param_shapes = zero_model_states[0].param_shapes
254
-
255
- # Reconstruction protocol:
256
- #
257
- # XXX: document this
258
-
259
- if debug:
260
- for i in range(world_size):
261
- for j in range(len(fp32_flat_groups[0])):
262
- print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
-
264
- # XXX: memory usage doubles here (zero2)
265
- num_param_groups = len(fp32_flat_groups[0])
266
- merged_single_partition_of_fp32_groups = []
267
- for i in range(num_param_groups):
268
- merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
- full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
- merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
- avail_numel = sum(
272
- [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
-
274
- if debug:
275
- wanted_params = sum([len(shapes) for shapes in param_shapes])
276
- wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
- # not asserting if there is a mismatch due to possible padding
278
- print(f"Have {avail_numel} numels to process.")
279
- print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
-
281
- # params
282
- # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
- # out-of-core computing solution
284
- total_numel = 0
285
- total_params = 0
286
- for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
- offset = 0
288
- avail_numel = full_single_fp32_vector.numel()
289
- for name, shape in shapes.items():
290
-
291
- unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
- total_numel += unpartitioned_numel
293
- total_params += 1
294
-
295
- if debug:
296
- print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
- state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
- offset += unpartitioned_numel
299
-
300
- # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
- # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
- # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
- # live optimizer object, so we are checking that the numbers are within the right range
304
- align_to = 2 * world_size
305
-
306
- def zero2_align(x):
307
- return align_to * math.ceil(x / align_to)
308
-
309
- if debug:
310
- print(f"original offset={offset}, avail_numel={avail_numel}")
311
-
312
- offset = zero2_align(offset)
313
- avail_numel = zero2_align(avail_numel)
314
-
315
- if debug:
316
- print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
-
318
- # Sanity check
319
- if offset != avail_numel:
320
- raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
-
322
- print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
-
324
-
325
- def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
- exclude_frozen_parameters):
327
- state_dict = OrderedDict()
328
-
329
- # buffers
330
- buffers = zero_model_states[0].buffers
331
- state_dict.update(buffers)
332
- if debug:
333
- print(f"added {len(buffers)} buffers")
334
-
335
- if not exclude_frozen_parameters:
336
- _zero2_merge_frozen_params(state_dict, zero_model_states)
337
-
338
- _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
-
340
- # recover shared parameters
341
- for pair in zero_model_states[0].shared_params:
342
- if pair[1] in state_dict:
343
- state_dict[pair[0]] = state_dict[pair[1]]
344
-
345
- return state_dict
346
-
347
-
348
- def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
- remainder = unpartitioned_numel % world_size
350
- padding_numel = (world_size - remainder) if remainder else 0
351
- partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
- return partitioned_numel, padding_numel
353
-
354
-
355
- def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
- if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
- return
358
-
359
- if debug:
360
- for i in range(world_size):
361
- num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
- print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
-
364
- frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
- wanted_params = len(frozen_param_shapes)
366
- wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
- avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
- print(f'Frozen params: Have {avail_numel} numels to process.')
369
- print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
-
371
- total_params = 0
372
- total_numel = 0
373
- for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
- total_params += 1
375
- unpartitioned_numel = shape.numel()
376
- total_numel += unpartitioned_numel
377
-
378
- param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
- state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
-
381
- partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
-
383
- if debug:
384
- print(
385
- f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
- )
387
-
388
- print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
-
390
-
391
- class GatheredTensor:
392
- """
393
- A pseudo tensor that collects partitioned weights.
394
- It is more memory efficient when there are multiple groups.
395
- """
396
-
397
- def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
- self.flat_groups = flat_groups
399
- self.flat_groups_offset = flat_groups_offset
400
- self.offset = offset
401
- self.partitioned_numel = partitioned_numel
402
- self.shape = shape
403
- self.dtype = self.flat_groups[0][0].dtype
404
-
405
- def contiguous(self):
406
- """
407
- Merge partitioned weights from flat_groups into a single tensor.
408
- """
409
- end_idx = self.offset + self.partitioned_numel
410
- world_size = len(self.flat_groups)
411
- pad_flat_param_chunks = []
412
-
413
- for rank_i in range(world_size):
414
- # for each rank, we need to collect weights from related group/groups
415
- flat_groups_at_rank_i = self.flat_groups[rank_i]
416
- start_group_id = None
417
- end_group_id = None
418
- for group_id in range(len(self.flat_groups_offset)):
419
- if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
- start_group_id = group_id
421
- if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
- end_group_id = group_id
423
- break
424
- # collect weights from related group/groups
425
- for group_id in range(start_group_id, end_group_id + 1):
426
- flat_tensor = flat_groups_at_rank_i[group_id]
427
- start_offset = self.offset - self.flat_groups_offset[group_id]
428
- end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
- pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
-
431
- # collect weights from all ranks
432
- pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
- param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
- return param
435
-
436
-
437
- def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
- param_shapes = zero_model_states[0].param_shapes
439
- avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
-
441
- # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
- # param, re-consolidating each param, while dealing with padding if any
443
-
444
- # merge list of dicts, preserving order
445
- param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
-
447
- if debug:
448
- for i in range(world_size):
449
- print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
-
451
- wanted_params = len(param_shapes)
452
- wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
- # not asserting if there is a mismatch due to possible padding
454
- avail_numel = fp32_flat_groups[0].numel() * world_size
455
- print(f"Trainable params: Have {avail_numel} numels to process.")
456
- print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
-
458
- # params
459
- # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
- # out-of-core computing solution
461
- offset = 0
462
- total_numel = 0
463
- total_params = 0
464
- flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
- for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
- unpartitioned_numel = shape.numel()
467
- total_numel += unpartitioned_numel
468
- total_params += 1
469
- partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
-
471
- if debug:
472
- print(
473
- f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
- )
475
-
476
- # memory efficient tensor
477
- tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
- state_dict[name] = tensor
479
- offset += partitioned_numel
480
-
481
- offset *= world_size
482
-
483
- # Sanity check
484
- if offset != avail_numel:
485
- raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
-
487
- print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
-
489
-
490
- def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
- exclude_frozen_parameters):
492
- state_dict = OrderedDict()
493
-
494
- # buffers
495
- buffers = zero_model_states[0].buffers
496
- state_dict.update(buffers)
497
- if debug:
498
- print(f"added {len(buffers)} buffers")
499
-
500
- if not exclude_frozen_parameters:
501
- _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
-
503
- _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
-
505
- # recover shared parameters
506
- for pair in zero_model_states[0].shared_params:
507
- if pair[1] in state_dict:
508
- state_dict[pair[0]] = state_dict[pair[1]]
509
-
510
- return state_dict
511
-
512
-
513
- def to_torch_tensor(state_dict, return_empty_tensor=False):
514
- """
515
- Convert state_dict of GatheredTensor to torch tensor
516
- """
517
- torch_state_dict = {}
518
- converted_tensors = {}
519
- for name, tensor in state_dict.items():
520
- tensor_id = id(tensor)
521
- if tensor_id in converted_tensors: # shared tensors
522
- shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
- torch_state_dict[name] = shared_tensor
524
- else:
525
- converted_tensors[tensor_id] = name
526
- if return_empty_tensor:
527
- torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
- else:
529
- torch_state_dict[name] = tensor.contiguous()
530
- return torch_state_dict
531
-
532
-
533
- def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
- tag=None,
535
- exclude_frozen_parameters=False,
536
- lazy_mode=False):
537
- """
538
- Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
- ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
- via a model hub.
541
-
542
- Args:
543
- - ``checkpoint_dir``: path to the desired checkpoint folder
544
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
- - ``exclude_frozen_parameters``: exclude frozen parameters
546
- - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
- Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
-
549
- Returns:
550
- - pytorch ``state_dict``
551
-
552
- A typical usage might be ::
553
-
554
- from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
- # do the training and checkpoint saving
556
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
- model = model.cpu() # move to cpu
558
- model.load_state_dict(state_dict)
559
- # submit to model hub or save the model to share with others
560
-
561
- In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
- application. i.e. you will need to re-initialize the deepspeed engine, since
563
- ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
-
565
- If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
-
567
- Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
- You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
- the checkpoint. Or you can load state_dict in lazy mode ::
570
-
571
- from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
- for name, lazy_tensor in state_dict.item():
574
- tensor = lazy_tensor.contiguous() # to cpu
575
- print(name, tensor)
576
- # del tensor to release memory if it no longer in use
577
- """
578
- if tag is None:
579
- latest_path = os.path.join(checkpoint_dir, 'latest')
580
- if os.path.isfile(latest_path):
581
- with open(latest_path, 'r') as fd:
582
- tag = fd.read().strip()
583
- else:
584
- raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
-
586
- ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
-
588
- if not os.path.isdir(ds_checkpoint_dir):
589
- raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
-
591
- state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
- if lazy_mode:
593
- return state_dict
594
- else:
595
- return to_torch_tensor(state_dict)
596
-
597
-
598
- def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
- output_dir,
600
- max_shard_size="5GB",
601
- safe_serialization=False,
602
- tag=None,
603
- exclude_frozen_parameters=False):
604
- """
605
- Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
- loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
-
608
- Args:
609
- - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
- - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
- - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
- - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
- - ``exclude_frozen_parameters``: exclude frozen parameters
615
- """
616
-
617
- # Dependency pre-check
618
- if safe_serialization:
619
- try:
620
- from safetensors.torch import save_file
621
- except ImportError:
622
- print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
- raise
624
- if max_shard_size is not None:
625
- try:
626
- from huggingface_hub import split_torch_state_dict_into_shards
627
- except ImportError:
628
- print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
- raise
630
-
631
- # Convert zero checkpoint to state_dict
632
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
- tag,
634
- exclude_frozen_parameters,
635
- lazy_mode=True)
636
-
637
- # Shard the model if it is too big.
638
- weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
- if max_shard_size is not None:
640
- filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
- # an memory-efficient approach for sharding
642
- empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
- state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
- filename_pattern=filename_pattern,
645
- max_shard_size=max_shard_size)
646
- else:
647
- from collections import namedtuple
648
- StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
- state_dict_split = StateDictSplit(is_sharded=False,
650
- filename_to_tensors={weights_name: list(state_dict.keys())})
651
-
652
- # Save the model by shard
653
- os.makedirs(output_dir, exist_ok=True)
654
- filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
- for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
- shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
- shard_state_dict = to_torch_tensor(shard_state_dict)
658
- output_path = os.path.join(output_dir, shard_file)
659
- if safe_serialization:
660
- save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
- else:
662
- torch.save(shard_state_dict, output_path)
663
- # release the memory of current shard
664
- for tensor_name in list(shard_state_dict.keys()):
665
- del state_dict[tensor_name]
666
- del shard_state_dict[tensor_name]
667
- del shard_state_dict
668
- gc.collect()
669
-
670
- # Save index if sharded
671
- if state_dict_split.is_sharded:
672
- index = {
673
- "metadata": state_dict_split.metadata,
674
- "weight_map": state_dict_split.tensor_to_filename,
675
- }
676
- save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
- save_index_file = os.path.join(output_dir, save_index_file)
678
- with open(save_index_file, "w", encoding="utf-8") as f:
679
- content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
- f.write(content)
681
-
682
-
683
- def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
- """
685
- 1. Put the provided model to cpu
686
- 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
- 3. Load it into the provided model
688
-
689
- Args:
690
- - ``model``: the model object to update
691
- - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
-
694
- Returns:
695
- - ``model`: modified model
696
-
697
- Make sure you have plenty of CPU memory available before you call this function. If you don't
698
- have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
- conveniently placed for you in the checkpoint folder.
700
-
701
- A typical usage might be ::
702
-
703
- from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
- model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
- # submit to model hub or save the model to share with others
706
-
707
- Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
- of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
- ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
-
711
- """
712
- logger.info("Extracting fp32 weights")
713
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
-
715
- logger.info("Overwriting model with fp32 weights")
716
- model = model.cpu()
717
- model.load_state_dict(state_dict, strict=False)
718
-
719
- return model
720
-
721
-
722
- if __name__ == "__main__":
723
- parser = argparse.ArgumentParser()
724
- parser.add_argument("checkpoint_dir",
725
- type=str,
726
- help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
- parser.add_argument("output_dir",
728
- type=str,
729
- help="directory to the pytorch fp32 state_dict output files"
730
- "(e.g. path/checkpoint-12-output/)")
731
- parser.add_argument(
732
- "--max_shard_size",
733
- type=str,
734
- default="5GB",
735
- help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
- "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
- "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
- "without CPU OOM issues.")
739
- parser.add_argument(
740
- "--safe_serialization",
741
- default=False,
742
- action='store_true',
743
- help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
- parser.add_argument("-t",
745
- "--tag",
746
- type=str,
747
- default=None,
748
- help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
- parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
- parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
- args = parser.parse_args()
752
-
753
- debug = args.debug
754
-
755
- convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
- args.output_dir,
757
- max_shard_size=args.max_shard_size,
758
- safe_serialization=args.safe_serialization,
759
- tag=args.tag,
760
- exclude_frozen_parameters=args.exclude_frozen_parameters)