Hugging Face's logo

RuoliuYang
/
analysis

Safetensors
Model card Files
xet
 Community
RuoliuYang commited on
Commit
74d6c21
·
verified ·
1 Parent(s): 19b765e

prune checkpoint-1000 (keep latest 3)

Browse files
Files changed (26) hide show
  1. size-250k/checkpoint-1000/added_tokens.json +0 -29
  2. size-250k/checkpoint-1000/chat_template.jinja +0 -7
  3. size-250k/checkpoint-1000/config.json +0 -144
  4. size-250k/checkpoint-1000/generation_config.json +0 -12
  5. size-250k/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +0 -3
  6. size-250k/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +0 -3
  7. size-250k/checkpoint-1000/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt +0 -3
  8. size-250k/checkpoint-1000/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt +0 -3
  9. size-250k/checkpoint-1000/global_step1000/mp_rank_00_model_states.pt +0 -3
  10. size-250k/checkpoint-1000/latest +0 -1
  11. size-250k/checkpoint-1000/merges.txt +0 -0
  12. size-250k/checkpoint-1000/model-00001-of-00004.safetensors +0 -3
  13. size-250k/checkpoint-1000/model-00002-of-00004.safetensors +0 -3
  14. size-250k/checkpoint-1000/model-00003-of-00004.safetensors +0 -3
  15. size-250k/checkpoint-1000/model-00004-of-00004.safetensors +0 -3
  16. size-250k/checkpoint-1000/model.safetensors.index.json +0 -737
  17. size-250k/checkpoint-1000/preprocessor_config.json +0 -37
  18. size-250k/checkpoint-1000/scheduler.pt +0 -3
  19. size-250k/checkpoint-1000/special_tokens_map.json +0 -31
  20. size-250k/checkpoint-1000/tokenizer.json +0 -3
  21. size-250k/checkpoint-1000/tokenizer_config.json +0 -249
  22. size-250k/checkpoint-1000/trainer_state.json +0 -1134
  23. size-250k/checkpoint-1000/training_args.bin +0 -3
  24. size-250k/checkpoint-1000/video_preprocessor_config.json +0 -43
  25. size-250k/checkpoint-1000/vocab.json +0 -0
  26. size-250k/checkpoint-1000/zero_to_fp32.py +0 -760
size-250k/checkpoint-1000/added_tokens.json DELETED
@@ -1,29 +0,0 @@
1
- {
2
- "</abs_vis_token>": 151667,
3
- "</observation>": 151669,
4
- "</tool_call>": 151658,
5
- "<abs_vis_token>": 151666,
6
- "<abs_vis_token_pad>": 151665,
7
- "<observation>": 151668,
8
- "<tool_call>": 151657,
9
- "<|box_end|>": 151649,
10
- "<|box_start|>": 151648,
11
- "<|endoftext|>": 151643,
12
- "<|file_sep|>": 151664,
13
- "<|fim_middle|>": 151660,
14
- "<|fim_pad|>": 151662,
15
- "<|fim_prefix|>": 151659,
16
- "<|fim_suffix|>": 151661,
17
- "<|im_end|>": 151645,
18
- "<|im_start|>": 151644,
19
- "<|image_pad|>": 151655,
20
- "<|object_ref_end|>": 151647,
21
- "<|object_ref_start|>": 151646,
22
- "<|quad_end|>": 151651,
23
- "<|quad_start|>": 151650,
24
- "<|repo_name|>": 151663,
25
- "<|video_pad|>": 151656,
26
- "<|vision_end|>": 151653,
27
- "<|vision_pad|>": 151654,
28
- "<|vision_start|>": 151652
29
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/chat_template.jinja DELETED
@@ -1,7 +0,0 @@
1
- {% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
2
- You are a helpful assistant.<|im_end|>
3
- {% endif %}<|im_start|>{{ message['role'] }}
4
- {% if message['content'] is string %}{{ message['content'] }}<|im_end|>
5
- {% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
6
- {% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
7
- {% endif %}
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/config.json DELETED
@@ -1,144 +0,0 @@
1
- {
2
- "answer_start_pattern": [
3
- 151644,
4
- 77091
5
- ],
6
- "architectures": [
7
- "Qwen2_5_VLForConditionalGeneration"
8
- ],
9
- "attention_dropout": 0.0,
10
- "bos_token_id": 151643,
11
- "eos_token_id": 151645,
12
- "hidden_act": "silu",
13
- "hidden_size": 3584,
14
- "image_token_id": 151655,
15
- "initializer_range": 0.02,
16
- "intermediate_size": 18944,
17
- "latent_end_id": 151667,
18
- "latent_start_id": 151666,
19
- "latent_token_id": 151665,
20
- "loss_type": "ForCausalLMLoss",
21
- "max_position_embeddings": 128000,
22
- "max_window_layers": 28,
23
- "model_type": "qwen2_5_vl",
24
- "num_attention_heads": 28,
25
- "num_hidden_layers": 28,
26
- "num_key_value_heads": 4,
27
- "rms_norm_eps": 1e-06,
28
- "rope_scaling": {
29
- "mrope_section": [
30
- 16,
31
- 24,
32
- 24
33
- ],
34
- "rope_type": "default",
35
- "type": "default"
36
- },
37
- "rope_theta": 1000000.0,
38
- "sliding_window": 32768,
39
- "stage": "sft_stage2",
40
- "text_config": {
41
- "architectures": [
42
- "Qwen2_5_VLForConditionalGeneration"
43
- ],
44
- "attention_dropout": 0.0,
45
- "bos_token_id": 151643,
46
- "eos_token_id": 151645,
47
- "hidden_act": "silu",
48
- "hidden_size": 3584,
49
- "image_token_id": null,
50
- "initializer_range": 0.02,
51
- "intermediate_size": 18944,
52
- "layer_types": [
53
- "full_attention",
54
- "full_attention",
55
- "full_attention",
56
- "full_attention",
57
- "full_attention",
58
- "full_attention",
59
- "full_attention",
60
- "full_attention",
61
- "full_attention",
62
- "full_attention",
63
- "full_attention",
64
- "full_attention",
65
- "full_attention",
66
- "full_attention",
67
- "full_attention",
68
- "full_attention",
69
- "full_attention",
70
- "full_attention",
71
- "full_attention",
72
- "full_attention",
73
- "full_attention",
74
- "full_attention",
75
- "full_attention",
76
- "full_attention",
77
- "full_attention",
78
- "full_attention",
79
- "full_attention",
80
- "full_attention"
81
- ],
82
- "max_position_embeddings": 128000,
83
- "max_window_layers": 28,
84
- "model_type": "qwen2_5_vl_text",
85
- "num_attention_heads": 28,
86
- "num_hidden_layers": 28,
87
- "num_key_value_heads": 4,
88
- "rms_norm_eps": 1e-06,
89
- "rope_scaling": {
90
- "mrope_section": [
91
- 16,
92
- 24,
93
- 24
94
- ],
95
- "rope_type": "default",
96
- "type": "default"
97
- },
98
- "rope_theta": 1000000.0,
99
- "sliding_window": null,
100
- "torch_dtype": "bfloat16",
101
- "use_cache": true,
102
- "use_sliding_window": false,
103
- "video_token_id": null,
104
- "vision_end_token_id": 151653,
105
- "vision_start_token_id": 151652,
106
- "vision_token_id": 151654,
107
- "vocab_size": 151670
108
- },
109
- "tie_word_embeddings": false,
110
- "torch_dtype": "bfloat16",
111
- "transformers_version": "4.54.0",
112
- "use_cache": false,
113
- "use_sliding_window": false,
114
- "video_token_id": 151656,
115
- "vision_config": {
116
- "depth": 32,
117
- "fullatt_block_indexes": [
118
- 7,
119
- 15,
120
- 23,
121
- 31
122
- ],
123
- "hidden_act": "silu",
124
- "hidden_size": 1280,
125
- "in_channels": 3,
126
- "in_chans": 3,
127
- "initializer_range": 0.02,
128
- "intermediate_size": 3420,
129
- "model_type": "qwen2_5_vl",
130
- "num_heads": 16,
131
- "out_hidden_size": 3584,
132
- "patch_size": 14,
133
- "spatial_merge_size": 2,
134
- "spatial_patch_size": 14,
135
- "temporal_patch_size": 2,
136
- "tokens_per_second": 2,
137
- "torch_dtype": "bfloat16",
138
- "window_size": 112
139
- },
140
- "vision_end_token_id": 151653,
141
- "vision_start_token_id": 151652,
142
- "vision_token_id": 151654,
143
- "vocab_size": 151670
144
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/generation_config.json DELETED
@@ -1,12 +0,0 @@
1
- {
2
- "bos_token_id": 151643,
3
- "do_sample": true,
4
- "eos_token_id": [
5
- 151645,
6
- 151643
7
- ],
8
- "pad_token_id": 151643,
9
- "repetition_penalty": 1.05,
10
- "temperature": 1e-06,
11
- "transformers_version": "4.54.0"
12
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4cb9f5c04e1e92a24f27a01c3f78913d6900f1397cd7e9cb023a1d99b1da0c6d
3
- size 11419196428
 
 
 
 
size-250k/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0bf4d70f6eef20bb6a38b17aa0f86aa86ba33d98b2ae561c90a1622377990409
3
- size 11419197708
 
 
 
 
size-250k/checkpoint-1000/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d9698542265077c352ba69f01e0c710bb0811218becafb7d5e20f56fb39b3439
3
- size 11419197772
 
 
 
 
size-250k/checkpoint-1000/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:1aa2d8029719dbbb811da137e7b274fa126d8a3a2990b4ddebe9f3b318e90b42
3
- size 11419197772
 
 
 
 
size-250k/checkpoint-1000/global_step1000/mp_rank_00_model_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:81eedd0967138273b62aec61ee82d26140564cc2d3f16c14204bedae4de757ff
3
- size 17932200534
 
 
 
 
size-250k/checkpoint-1000/latest DELETED
@@ -1 +0,0 @@
1
- global_step1000
 
 
size-250k/checkpoint-1000/merges.txt DELETED
The diff for this file is too large to render. See raw diff
 
size-250k/checkpoint-1000/model-00001-of-00004.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a5d72f2e317e5876427be1738f0678a6664f86a00eaf4513d28b415529c66079
3
- size 4965419112
 
 
 
 
size-250k/checkpoint-1000/model-00002-of-00004.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:aaee93971d63f63810de0d7d063f8ef09ee5095a91ac5f2f79afc9a3a89d6040
3
- size 4991495816
 
 
 
 
size-250k/checkpoint-1000/model-00003-of-00004.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d29b2d958a30423e17adc4661acb6b7d556abd91c8cdb4161ed78f9cc6143de5
3
- size 4932751040
 
 
 
 
size-250k/checkpoint-1000/model-00004-of-00004.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:2663c9c9329a570334d709e897fd26ce49c30bc34be890c9d8975599b1ee485a
3
- size 1689100192
 
 
 
 
size-250k/checkpoint-1000/model.safetensors.index.json DELETED
@@ -1,737 +0,0 @@
1
- {
2
- "metadata": {
3
- "total_parameters": 8289342464,
4
- "total_size": 16578684928
5
- },
6
- "weight_map": {
7
- "lm_head.weight": "model-00004-of-00004.safetensors",
8
- "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
9
- "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
10
- "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
11
- "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
12
- "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
13
- "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
14
- "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
15
- "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
16
- "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
17
- "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
18
- "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
19
- "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
20
- "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
21
- "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
22
- "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
23
- "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
24
- "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
25
- "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
26
- "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
27
- "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
28
- "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
29
- "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
30
- "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
31
- "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
32
- "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
33
- "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
34
- "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
35
- "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
36
- "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
37
- "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
38
- "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
39
- "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
40
- "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
41
- "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
42
- "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
43
- "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
44
- "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
45
- "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
46
- "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
47
- "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
48
- "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
49
- "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
50
- "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
51
- "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
52
- "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
53
- "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
54
- "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
55
- "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
56
- "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
57
- "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
58
- "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
59
- "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
60
- "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
61
- "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
62
- "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
63
- "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
64
- "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
65
- "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
66
- "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
67
- "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
68
- "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
69
- "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
70
- "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
71
- "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
72
- "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
73
- "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
74
- "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
75
- "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
76
- "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
77
- "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
78
- "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
79
- "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
80
- "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
81
- "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
82
- "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
83
- "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
84
- "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
85
- "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
86
- "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
87
- "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
88
- "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
89
- "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
90
- "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
91
- "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
92
- "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
93
- "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
94
- "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
95
- "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
96
- "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
97
- "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
98
- "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
99
- "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
100
- "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
101
- "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
102
- "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
103
- "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
104
- "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
105
- "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
106
- "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
107
- "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
108
- "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
109
- "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
110
- "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
111
- "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
112
- "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
113
- "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
114
- "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
115
- "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
116
- "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
117
- "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
118
- "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
119
- "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
120
- "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
121
- "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
122
- "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
123
- "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
124
- "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
125
- "model.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
126
- "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
127
- "model.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
128
- "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
129
- "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
130
- "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
131
- "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
132
- "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
133
- "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
134
- "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
135
- "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
136
- "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
137
- "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
138
- "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
139
- "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
140
- "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
141
- "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
142
- "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
143
- "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
144
- "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
145
- "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
146
- "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
147
- "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
148
- "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
149
- "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
150
- "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
151
- "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
152
- "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
153
- "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
154
- "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
155
- "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
156
- "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
157
- "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
158
- "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
159
- "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
160
- "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
161
- "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
162
- "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
163
- "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
164
- "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
165
- "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
166
- "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
167
- "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
168
- "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
169
- "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
170
- "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
171
- "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
172
- "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
173
- "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
174
- "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
175
- "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
176
- "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
177
- "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
178
- "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
179
- "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
180
- "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
181
- "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
182
- "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
183
- "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
184
- "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
185
- "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
186
- "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
187
- "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
188
- "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
189
- "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
190
- "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
191
- "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
192
- "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
193
- "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
194
- "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
195
- "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
196
- "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
197
- "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
198
- "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
199
- "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
200
- "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
201
- "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
202
- "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
203
- "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
204
- "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
205
- "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
206
- "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
207
- "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
208
- "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
209
- "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
210
- "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
211
- "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
212
- "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
213
- "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
214
- "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
215
- "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
216
- "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
217
- "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
218
- "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
219
- "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
220
- "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
221
- "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
222
- "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
223
- "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
224
- "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
225
- "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
226
- "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
227
- "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
228
- "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
229
- "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
230
- "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
231
- "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
232
- "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
233
- "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
234
- "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
235
- "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
236
- "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
237
- "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
238
- "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
239
- "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
240
- "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
241
- "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
242
- "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
243
- "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
244
- "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
245
- "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
246
- "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
247
- "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
248
- "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
249
- "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
250
- "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
251
- "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
252
- "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
253
- "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
254
- "model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
255
- "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
256
- "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
257
- "model.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
258
- "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
259
- "model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
260
- "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
261
- "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
262
- "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
263
- "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
264
- "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
265
- "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
266
- "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
267
- "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
268
- "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
269
- "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
270
- "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
271
- "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
272
- "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
273
- "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
274
- "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
275
- "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
276
- "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
277
- "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
278
- "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
279
- "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
280
- "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
281
- "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
282
- "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
283
- "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
284
- "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
285
- "model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
286
- "model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
287
- "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
288
- "model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
289
- "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
290
- "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
291
- "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
292
- "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
293
- "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
294
- "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
295
- "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
296
- "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
297
- "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
298
- "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
299
- "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
300
- "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
301
- "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
302
- "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
303
- "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
304
- "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
305
- "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
306
- "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
307
- "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
308
- "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
309
- "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
310
- "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
311
- "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
312
- "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
313
- "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
314
- "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
315
- "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
316
- "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
317
- "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
318
- "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
319
- "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
320
- "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
321
- "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
322
- "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
323
- "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
324
- "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
325
- "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
326
- "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
327
- "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
328
- "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
329
- "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
330
- "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
331
- "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
332
- "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
333
- "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
334
- "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
335
- "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
336
- "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
337
- "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
338
- "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
339
- "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
340
- "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
341
- "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
342
- "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
343
- "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
344
- "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
345
- "model.norm.weight": "model-00004-of-00004.safetensors",
346
- "visual.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
347
- "visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
348
- "visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
349
- "visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
350
- "visual.blocks.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
351
- "visual.blocks.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
352
- "visual.blocks.0.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
353
- "visual.blocks.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
354
- "visual.blocks.0.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
355
- "visual.blocks.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
356
- "visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
357
- "visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
358
- "visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
359
- "visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
360
- "visual.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
361
- "visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
362
- "visual.blocks.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
363
- "visual.blocks.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
364
- "visual.blocks.1.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
365
- "visual.blocks.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
366
- "visual.blocks.1.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
367
- "visual.blocks.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
368
- "visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
369
- "visual.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
370
- "visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
371
- "visual.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
372
- "visual.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
373
- "visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
374
- "visual.blocks.10.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
375
- "visual.blocks.10.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
376
- "visual.blocks.10.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
377
- "visual.blocks.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
378
- "visual.blocks.10.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
379
- "visual.blocks.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
380
- "visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
381
- "visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
382
- "visual.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
383
- "visual.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
384
- "visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
385
- "visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
386
- "visual.blocks.11.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
387
- "visual.blocks.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
388
- "visual.blocks.11.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
389
- "visual.blocks.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
390
- "visual.blocks.11.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
391
- "visual.blocks.11.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
392
- "visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
393
- "visual.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
394
- "visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
395
- "visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
396
- "visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
397
- "visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
398
- "visual.blocks.12.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
399
- "visual.blocks.12.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
400
- "visual.blocks.12.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
401
- "visual.blocks.12.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
402
- "visual.blocks.12.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
403
- "visual.blocks.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
404
- "visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
405
- "visual.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
406
- "visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
407
- "visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
408
- "visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
409
- "visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
410
- "visual.blocks.13.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
411
- "visual.blocks.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
412
- "visual.blocks.13.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
413
- "visual.blocks.13.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
414
- "visual.blocks.13.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
415
- "visual.blocks.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
416
- "visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
417
- "visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
418
- "visual.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
419
- "visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
420
- "visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
421
- "visual.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
422
- "visual.blocks.14.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
423
- "visual.blocks.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
424
- "visual.blocks.14.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
425
- "visual.blocks.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
426
- "visual.blocks.14.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
427
- "visual.blocks.14.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
428
- "visual.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
429
- "visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
430
- "visual.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
431
- "visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
432
- "visual.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
433
- "visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
434
- "visual.blocks.15.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
435
- "visual.blocks.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
436
- "visual.blocks.15.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
437
- "visual.blocks.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
438
- "visual.blocks.15.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
439
- "visual.blocks.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
440
- "visual.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
441
- "visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
442
- "visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
443
- "visual.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
444
- "visual.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
445
- "visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
446
- "visual.blocks.16.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
447
- "visual.blocks.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
448
- "visual.blocks.16.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
449
- "visual.blocks.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
450
- "visual.blocks.16.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
451
- "visual.blocks.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
452
- "visual.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
453
- "visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
454
- "visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
455
- "visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
456
- "visual.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
457
- "visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
458
- "visual.blocks.17.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
459
- "visual.blocks.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
460
- "visual.blocks.17.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
461
- "visual.blocks.17.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
462
- "visual.blocks.17.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
463
- "visual.blocks.17.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
464
- "visual.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
465
- "visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
466
- "visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
467
- "visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
468
- "visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
469
- "visual.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
470
- "visual.blocks.18.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
471
- "visual.blocks.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
472
- "visual.blocks.18.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
473
- "visual.blocks.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
474
- "visual.blocks.18.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
475
- "visual.blocks.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
476
- "visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
477
- "visual.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
478
- "visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
479
- "visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
480
- "visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
481
- "visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
482
- "visual.blocks.19.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
483
- "visual.blocks.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
484
- "visual.blocks.19.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
485
- "visual.blocks.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
486
- "visual.blocks.19.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
487
- "visual.blocks.19.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
488
- "visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
489
- "visual.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
490
- "visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
491
- "visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
492
- "visual.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
493
- "visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
494
- "visual.blocks.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
495
- "visual.blocks.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
496
- "visual.blocks.2.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
497
- "visual.blocks.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
498
- "visual.blocks.2.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
499
- "visual.blocks.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
500
- "visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
501
- "visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
502
- "visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
503
- "visual.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
504
- "visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
505
- "visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
506
- "visual.blocks.20.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
507
- "visual.blocks.20.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
508
- "visual.blocks.20.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
509
- "visual.blocks.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
510
- "visual.blocks.20.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
511
- "visual.blocks.20.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
512
- "visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
513
- "visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
514
- "visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
515
- "visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
516
- "visual.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
517
- "visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
518
- "visual.blocks.21.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
519
- "visual.blocks.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
520
- "visual.blocks.21.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
521
- "visual.blocks.21.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
522
- "visual.blocks.21.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
523
- "visual.blocks.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
524
- "visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
525
- "visual.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
526
- "visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
527
- "visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
528
- "visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
529
- "visual.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
530
- "visual.blocks.22.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
531
- "visual.blocks.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
532
- "visual.blocks.22.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
533
- "visual.blocks.22.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
534
- "visual.blocks.22.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
535
- "visual.blocks.22.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
536
- "visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
537
- "visual.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
538
- "visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
539
- "visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
540
- "visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
541
- "visual.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
542
- "visual.blocks.23.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
543
- "visual.blocks.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
544
- "visual.blocks.23.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
545
- "visual.blocks.23.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
546
- "visual.blocks.23.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
547
- "visual.blocks.23.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
548
- "visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
549
- "visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
550
- "visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
551
- "visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
552
- "visual.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
553
- "visual.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
554
- "visual.blocks.24.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
555
- "visual.blocks.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
556
- "visual.blocks.24.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
557
- "visual.blocks.24.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
558
- "visual.blocks.24.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
559
- "visual.blocks.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
560
- "visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
561
- "visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
562
- "visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
563
- "visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
564
- "visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
565
- "visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
566
- "visual.blocks.25.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
567
- "visual.blocks.25.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
568
- "visual.blocks.25.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
569
- "visual.blocks.25.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
570
- "visual.blocks.25.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
571
- "visual.blocks.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
572
- "visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
573
- "visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
574
- "visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
575
- "visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
576
- "visual.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
577
- "visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
578
- "visual.blocks.26.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
579
- "visual.blocks.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
580
- "visual.blocks.26.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
581
- "visual.blocks.26.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
582
- "visual.blocks.26.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
583
- "visual.blocks.26.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
584
- "visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
585
- "visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
586
- "visual.blocks.27.attn.proj.bias": "model-00001-of-00004.safetensors",
587
- "visual.blocks.27.attn.proj.weight": "model-00001-of-00004.safetensors",
588
- "visual.blocks.27.attn.qkv.bias": "model-00001-of-00004.safetensors",
589
- "visual.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors",
590
- "visual.blocks.27.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
591
- "visual.blocks.27.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
592
- "visual.blocks.27.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
593
- "visual.blocks.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
594
- "visual.blocks.27.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
595
- "visual.blocks.27.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
596
- "visual.blocks.27.norm1.weight": "model-00001-of-00004.safetensors",
597
- "visual.blocks.27.norm2.weight": "model-00001-of-00004.safetensors",
598
- "visual.blocks.28.attn.proj.bias": "model-00001-of-00004.safetensors",
599
- "visual.blocks.28.attn.proj.weight": "model-00001-of-00004.safetensors",
600
- "visual.blocks.28.attn.qkv.bias": "model-00001-of-00004.safetensors",
601
- "visual.blocks.28.attn.qkv.weight": "model-00001-of-00004.safetensors",
602
- "visual.blocks.28.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
603
- "visual.blocks.28.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
604
- "visual.blocks.28.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
605
- "visual.blocks.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
606
- "visual.blocks.28.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
607
- "visual.blocks.28.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
608
- "visual.blocks.28.norm1.weight": "model-00001-of-00004.safetensors",
609
- "visual.blocks.28.norm2.weight": "model-00001-of-00004.safetensors",
610
- "visual.blocks.29.attn.proj.bias": "model-00001-of-00004.safetensors",
611
- "visual.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors",
612
- "visual.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors",
613
- "visual.blocks.29.attn.qkv.weight": "model-00001-of-00004.safetensors",
614
- "visual.blocks.29.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
615
- "visual.blocks.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
616
- "visual.blocks.29.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
617
- "visual.blocks.29.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
618
- "visual.blocks.29.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
619
- "visual.blocks.29.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
620
- "visual.blocks.29.norm1.weight": "model-00001-of-00004.safetensors",
621
- "visual.blocks.29.norm2.weight": "model-00001-of-00004.safetensors",
622
- "visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
623
- "visual.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
624
- "visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
625
- "visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
626
- "visual.blocks.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
627
- "visual.blocks.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
628
- "visual.blocks.3.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
629
- "visual.blocks.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
630
- "visual.blocks.3.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
631
- "visual.blocks.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
632
- "visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
633
- "visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
634
- "visual.blocks.30.attn.proj.bias": "model-00001-of-00004.safetensors",
635
- "visual.blocks.30.attn.proj.weight": "model-00001-of-00004.safetensors",
636
- "visual.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors",
637
- "visual.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors",
638
- "visual.blocks.30.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
639
- "visual.blocks.30.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
640
- "visual.blocks.30.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
641
- "visual.blocks.30.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
642
- "visual.blocks.30.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
643
- "visual.blocks.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
644
- "visual.blocks.30.norm1.weight": "model-00001-of-00004.safetensors",
645
- "visual.blocks.30.norm2.weight": "model-00001-of-00004.safetensors",
646
- "visual.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors",
647
- "visual.blocks.31.attn.proj.weight": "model-00001-of-00004.safetensors",
648
- "visual.blocks.31.attn.qkv.bias": "model-00001-of-00004.safetensors",
649
- "visual.blocks.31.attn.qkv.weight": "model-00001-of-00004.safetensors",
650
- "visual.blocks.31.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
651
- "visual.blocks.31.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
652
- "visual.blocks.31.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
653
- "visual.blocks.31.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
654
- "visual.blocks.31.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
655
- "visual.blocks.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
656
- "visual.blocks.31.norm1.weight": "model-00001-of-00004.safetensors",
657
- "visual.blocks.31.norm2.weight": "model-00001-of-00004.safetensors",
658
- "visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
659
- "visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
660
- "visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
661
- "visual.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
662
- "visual.blocks.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
663
- "visual.blocks.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
664
- "visual.blocks.4.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
665
- "visual.blocks.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
666
- "visual.blocks.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
667
- "visual.blocks.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
668
- "visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
669
- "visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
670
- "visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
671
- "visual.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
672
- "visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
673
- "visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
674
- "visual.blocks.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
675
- "visual.blocks.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
676
- "visual.blocks.5.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
677
- "visual.blocks.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
678
- "visual.blocks.5.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
679
- "visual.blocks.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
680
- "visual.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
681
- "visual.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
682
- "visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
683
- "visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
684
- "visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
685
- "visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
686
- "visual.blocks.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
687
- "visual.blocks.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
688
- "visual.blocks.6.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
689
- "visual.blocks.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
690
- "visual.blocks.6.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
691
- "visual.blocks.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
692
- "visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
693
- "visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
694
- "visual.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
695
- "visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
696
- "visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
697
- "visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
698
- "visual.blocks.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
699
- "visual.blocks.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
700
- "visual.blocks.7.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
701
- "visual.blocks.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
702
- "visual.blocks.7.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
703
- "visual.blocks.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
704
- "visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
705
- "visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
706
- "visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
707
- "visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
708
- "visual.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
709
- "visual.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
710
- "visual.blocks.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
711
- "visual.blocks.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
712
- "visual.blocks.8.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
713
- "visual.blocks.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
714
- "visual.blocks.8.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
715
- "visual.blocks.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
716
- "visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
717
- "visual.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
718
- "visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
719
- "visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
720
- "visual.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
721
- "visual.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
722
- "visual.blocks.9.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
723
- "visual.blocks.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
724
- "visual.blocks.9.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
725
- "visual.blocks.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
726
- "visual.blocks.9.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
727
- "visual.blocks.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
728
- "visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
729
- "visual.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
730
- "visual.merger.ln_q.weight": "model-00001-of-00004.safetensors",
731
- "visual.merger.mlp.0.bias": "model-00001-of-00004.safetensors",
732
- "visual.merger.mlp.0.weight": "model-00001-of-00004.safetensors",
733
- "visual.merger.mlp.2.bias": "model-00001-of-00004.safetensors",
734
- "visual.merger.mlp.2.weight": "model-00001-of-00004.safetensors",
735
- "visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors"
736
- }
737
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/preprocessor_config.json DELETED
@@ -1,37 +0,0 @@
1
- {
2
- "crop_size": null,
3
- "data_format": "channels_first",
4
- "default_to_square": true,
5
- "device": null,
6
- "disable_grouping": null,
7
- "do_center_crop": null,
8
- "do_convert_rgb": true,
9
- "do_normalize": true,
10
- "do_rescale": true,
11
- "do_resize": true,
12
- "image_mean": [
13
- 0.48145466,
14
- 0.4578275,
15
- 0.40821073
16
- ],
17
- "image_processor_type": "Qwen2VLImageProcessorFast",
18
- "image_std": [
19
- 0.26862954,
20
- 0.26130258,
21
- 0.27577711
22
- ],
23
- "input_data_format": null,
24
- "max_pixels": 12845056,
25
- "merge_size": 2,
26
- "min_pixels": 3136,
27
- "patch_size": 14,
28
- "processor_class": "Qwen2_5_VLProcessor",
29
- "resample": 3,
30
- "rescale_factor": 0.00392156862745098,
31
- "return_tensors": null,
32
- "size": {
33
- "longest_edge": 12845056,
34
- "shortest_edge": 3136
35
- },
36
- "temporal_patch_size": 2
37
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:54aa94eea3012bfcb425f9f43c8604ad7fc29603945325a14deb3107d9bb8c76
3
- size 1064
 
 
 
 
size-250k/checkpoint-1000/special_tokens_map.json DELETED
@@ -1,31 +0,0 @@
1
- {
2
- "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>",
5
- "<|object_ref_start|>",
6
- "<|object_ref_end|>",
7
- "<|box_start|>",
8
- "<|box_end|>",
9
- "<|quad_start|>",
10
- "<|quad_end|>",
11
- "<|vision_start|>",
12
- "<|vision_end|>",
13
- "<|vision_pad|>",
14
- "<|image_pad|>",
15
- "<|video_pad|>"
16
- ],
17
- "eos_token": {
18
- "content": "<|im_end|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- },
24
- "pad_token": {
25
- "content": "<|endoftext|>",
26
- "lstrip": false,
27
- "normalized": false,
28
- "rstrip": false,
29
- "single_word": false
30
- }
31
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4c9f7c087bb10192b30b4697b05fdaec59883791f4f6defca36d4e8d9891538e
3
- size 11422858
 
 
 
 
size-250k/checkpoint-1000/tokenizer_config.json DELETED
@@ -1,249 +0,0 @@
1
- {
2
- "add_bos_token": false,
3
- "add_prefix_space": false,
4
- "added_tokens_decoder": {
5
- "151643": {
6
- "content": "<|endoftext|>",
7
- "lstrip": false,
8
- "normalized": false,
9
- "rstrip": false,
10
- "single_word": false,
11
- "special": true
12
- },
13
- "151644": {
14
- "content": "<|im_start|>",
15
- "lstrip": false,
16
- "normalized": false,
17
- "rstrip": false,
18
- "single_word": false,
19
- "special": true
20
- },
21
- "151645": {
22
- "content": "<|im_end|>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false,
27
- "special": true
28
- },
29
- "151646": {
30
- "content": "<|object_ref_start|>",
31
- "lstrip": false,
32
- "normalized": false,
33
- "rstrip": false,
34
- "single_word": false,
35
- "special": true
36
- },
37
- "151647": {
38
- "content": "<|object_ref_end|>",
39
- "lstrip": false,
40
- "normalized": false,
41
- "rstrip": false,
42
- "single_word": false,
43
- "special": true
44
- },
45
- "151648": {
46
- "content": "<|box_start|>",
47
- "lstrip": false,
48
- "normalized": false,
49
- "rstrip": false,
50
- "single_word": false,
51
- "special": true
52
- },
53
- "151649": {
54
- "content": "<|box_end|>",
55
- "lstrip": false,
56
- "normalized": false,
57
- "rstrip": false,
58
- "single_word": false,
59
- "special": true
60
- },
61
- "151650": {
62
- "content": "<|quad_start|>",
63
- "lstrip": false,
64
- "normalized": false,
65
- "rstrip": false,
66
- "single_word": false,
67
- "special": true
68
- },
69
- "151651": {
70
- "content": "<|quad_end|>",
71
- "lstrip": false,
72
- "normalized": false,
73
- "rstrip": false,
74
- "single_word": false,
75
- "special": true
76
- },
77
- "151652": {
78
- "content": "<|vision_start|>",
79
- "lstrip": false,
80
- "normalized": false,
81
- "rstrip": false,
82
- "single_word": false,
83
- "special": true
84
- },
85
- "151653": {
86
- "content": "<|vision_end|>",
87
- "lstrip": false,
88
- "normalized": false,
89
- "rstrip": false,
90
- "single_word": false,
91
- "special": true
92
- },
93
- "151654": {
94
- "content": "<|vision_pad|>",
95
- "lstrip": false,
96
- "normalized": false,
97
- "rstrip": false,
98
- "single_word": false,
99
- "special": true
100
- },
101
- "151655": {
102
- "content": "<|image_pad|>",
103
- "lstrip": false,
104
- "normalized": false,
105
- "rstrip": false,
106
- "single_word": false,
107
- "special": true
108
- },
109
- "151656": {
110
- "content": "<|video_pad|>",
111
- "lstrip": false,
112
- "normalized": false,
113
- "rstrip": false,
114
- "single_word": false,
115
- "special": true
116
- },
117
- "151657": {
118
- "content": "<tool_call>",
119
- "lstrip": false,
120
- "normalized": false,
121
- "rstrip": false,
122
- "single_word": false,
123
- "special": false
124
- },
125
- "151658": {
126
- "content": "</tool_call>",
127
- "lstrip": false,
128
- "normalized": false,
129
- "rstrip": false,
130
- "single_word": false,
131
- "special": false
132
- },
133
- "151659": {
134
- "content": "<|fim_prefix|>",
135
- "lstrip": false,
136
- "normalized": false,
137
- "rstrip": false,
138
- "single_word": false,
139
- "special": false
140
- },
141
- "151660": {
142
- "content": "<|fim_middle|>",
143
- "lstrip": false,
144
- "normalized": false,
145
- "rstrip": false,
146
- "single_word": false,
147
- "special": false
148
- },
149
- "151661": {
150
- "content": "<|fim_suffix|>",
151
- "lstrip": false,
152
- "normalized": false,
153
- "rstrip": false,
154
- "single_word": false,
155
- "special": false
156
- },
157
- "151662": {
158
- "content": "<|fim_pad|>",
159
- "lstrip": false,
160
- "normalized": false,
161
- "rstrip": false,
162
- "single_word": false,
163
- "special": false
164
- },
165
- "151663": {
166
- "content": "<|repo_name|>",
167
- "lstrip": false,
168
- "normalized": false,
169
- "rstrip": false,
170
- "single_word": false,
171
- "special": false
172
- },
173
- "151664": {
174
- "content": "<|file_sep|>",
175
- "lstrip": false,
176
- "normalized": false,
177
- "rstrip": false,
178
- "single_word": false,
179
- "special": false
180
- },
181
- "151665": {
182
- "content": "<abs_vis_token_pad>",
183
- "lstrip": false,
184
- "normalized": false,
185
- "rstrip": false,
186
- "single_word": false,
187
- "special": true
188
- },
189
- "151666": {
190
- "content": "<abs_vis_token>",
191
- "lstrip": false,
192
- "normalized": false,
193
- "rstrip": false,
194
- "single_word": false,
195
- "special": true
196
- },
197
- "151667": {
198
- "content": "</abs_vis_token>",
199
- "lstrip": false,
200
- "normalized": false,
201
- "rstrip": false,
202
- "single_word": false,
203
- "special": true
204
- },
205
- "151668": {
206
- "content": "<observation>",
207
- "lstrip": false,
208
- "normalized": false,
209
- "rstrip": false,
210
- "single_word": false,
211
- "special": true
212
- },
213
- "151669": {
214
- "content": "</observation>",
215
- "lstrip": false,
216
- "normalized": false,
217
- "rstrip": false,
218
- "single_word": false,
219
- "special": true
220
- }
221
- },
222
- "additional_special_tokens": [
223
- "<|im_start|>",
224
- "<|im_end|>",
225
- "<|object_ref_start|>",
226
- "<|object_ref_end|>",
227
- "<|box_start|>",
228
- "<|box_end|>",
229
- "<|quad_start|>",
230
- "<|quad_end|>",
231
- "<|vision_start|>",
232
- "<|vision_end|>",
233
- "<|vision_pad|>",
234
- "<|image_pad|>",
235
- "<|video_pad|>"
236
- ],
237
- "bos_token": null,
238
- "clean_up_tokenization_spaces": false,
239
- "eos_token": "<|im_end|>",
240
- "errors": "replace",
241
- "extra_special_tokens": {},
242
- "model_max_length": 131072,
243
- "pad_token": "<|endoftext|>",
244
- "processor_class": "Qwen2_5_VLProcessor",
245
- "split_special_tokens": false,
246
- "tokenizer_class": "Qwen2Tokenizer",
247
- "unk_token": null,
248
- "use_fast": true
249
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/trainer_state.json DELETED
@@ -1,1134 +0,0 @@
1
- {
2
- "best_global_step": null,
3
- "best_metric": null,
4
- "best_model_checkpoint": null,
5
- "epoch": 0.5197674040866712,
6
- "eval_steps": 500,
7
- "global_step": 1000,
8
- "is_hyper_param_search": false,
9
- "is_local_process_zero": true,
10
- "is_world_process_zero": true,
11
- "log_history": [
12
- {
13
- "alignment_loss": 0.818257,
14
- "epoch": 0.005197674040866712,
15
- "grad_norm": 10616.4033203125,
16
- "learning_rate": 9e-06,
17
- "loss": 600.6838,
18
- "mean_token_accuracy": 0.6212009839713574,
19
- "num_tokens": 1084700.0,
20
- "step": 10,
21
- "teacher_ce_loss": 37.7183
22
- },
23
- {
24
- "alignment_loss": 0.894285,
25
- "epoch": 0.010395348081733424,
26
- "grad_norm": 3175.2373046875,
27
- "learning_rate": 9.97655028660761e-06,
28
- "loss": 163.3203,
29
- "mean_token_accuracy": 0.7600021116435528,
30
- "num_tokens": 2173682.0,
31
- "step": 20,
32
- "teacher_ce_loss": 9.920976
33
- },
34
- {
35
- "alignment_loss": 0.892784,
36
- "epoch": 0.015593022122600136,
37
- "grad_norm": 3032.060546875,
38
- "learning_rate": 9.950495049504951e-06,
39
- "loss": 112.184,
40
- "mean_token_accuracy": 0.7797912888228893,
41
- "num_tokens": 3254544.0,
42
- "step": 30,
43
- "teacher_ce_loss": 6.994861
44
- },
45
- {
46
- "alignment_loss": 0.871766,
47
- "epoch": 0.02079069616346685,
48
- "grad_norm": 2499.0107421875,
49
- "learning_rate": 9.924439812402293e-06,
50
- "loss": 80.4524,
51
- "mean_token_accuracy": 0.8553705904632807,
52
- "num_tokens": 4366899.0,
53
- "step": 40,
54
- "teacher_ce_loss": 4.613683
55
- },
56
- {
57
- "alignment_loss": 0.839332,
58
- "epoch": 0.02598837020433356,
59
- "grad_norm": 844.4440307617188,
60
- "learning_rate": 9.898384575299636e-06,
61
- "loss": 55.5309,
62
- "mean_token_accuracy": 0.8646903920918703,
63
- "num_tokens": 5454040.0,
64
- "step": 50,
65
- "teacher_ce_loss": 3.528176
66
- },
67
- {
68
- "alignment_loss": 0.852953,
69
- "epoch": 0.031186044245200273,
70
- "grad_norm": 493.1343994140625,
71
- "learning_rate": 9.872329338196979e-06,
72
- "loss": 47.6412,
73
- "mean_token_accuracy": 0.8714054178446531,
74
- "num_tokens": 6535542.0,
75
- "step": 60,
76
- "teacher_ce_loss": 2.624911
77
- },
78
- {
79
- "alignment_loss": 0.850321,
80
- "epoch": 0.036383718286066985,
81
- "grad_norm": 502.66070556640625,
82
- "learning_rate": 9.84627410109432e-06,
83
- "loss": 45.5853,
84
- "mean_token_accuracy": 0.8716515514999628,
85
- "num_tokens": 7587360.0,
86
- "step": 70,
87
- "teacher_ce_loss": 2.632225
88
- },
89
- {
90
- "alignment_loss": 0.855101,
91
- "epoch": 0.0415813923269337,
92
- "grad_norm": 432.3569030761719,
93
- "learning_rate": 9.820218863991662e-06,
94
- "loss": 43.1104,
95
- "mean_token_accuracy": 0.8750173676759004,
96
- "num_tokens": 8677343.0,
97
- "step": 80,
98
- "teacher_ce_loss": 2.711963
99
- },
100
- {
101
- "alignment_loss": 0.835686,
102
- "epoch": 0.04677906636780041,
103
- "grad_norm": 481.1368713378906,
104
- "learning_rate": 9.794163626889005e-06,
105
- "loss": 42.8872,
106
- "mean_token_accuracy": 0.8726172130554914,
107
- "num_tokens": 9750881.0,
108
- "step": 90,
109
- "teacher_ce_loss": 2.858154
110
- },
111
- {
112
- "alignment_loss": 0.819627,
113
- "epoch": 0.05197674040866712,
114
- "grad_norm": 509.544921875,
115
- "learning_rate": 9.768108389786348e-06,
116
- "loss": 46.8516,
117
- "mean_token_accuracy": 0.8589316807687283,
118
- "num_tokens": 10855717.0,
119
- "step": 100,
120
- "teacher_ce_loss": 2.724989
121
- },
122
- {
123
- "alignment_loss": 0.848128,
124
- "epoch": 0.057174414449533834,
125
- "grad_norm": 459.5321044921875,
126
- "learning_rate": 9.74205315268369e-06,
127
- "loss": 44.1503,
128
- "mean_token_accuracy": 0.8717804063111544,
129
- "num_tokens": 11945834.0,
130
- "step": 110,
131
- "teacher_ce_loss": 2.576751
132
- },
133
- {
134
- "alignment_loss": 0.80412,
135
- "epoch": 0.062372088490400546,
136
- "grad_norm": 447.5848388671875,
137
- "learning_rate": 9.715997915581033e-06,
138
- "loss": 44.0393,
139
- "mean_token_accuracy": 0.8642505194991827,
140
- "num_tokens": 13051330.0,
141
- "step": 120,
142
- "teacher_ce_loss": 2.73768
143
- },
144
- {
145
- "alignment_loss": 0.822922,
146
- "epoch": 0.06756976253126726,
147
- "grad_norm": 439.0763244628906,
148
- "learning_rate": 9.689942678478375e-06,
149
- "loss": 44.999,
150
- "mean_token_accuracy": 0.8639923941344023,
151
- "num_tokens": 14127240.0,
152
- "step": 130,
153
- "teacher_ce_loss": 3.093184
154
- },
155
- {
156
- "alignment_loss": 0.804555,
157
- "epoch": 0.07276743657213397,
158
- "grad_norm": 446.28741455078125,
159
- "learning_rate": 9.663887441375718e-06,
160
- "loss": 42.6522,
161
- "mean_token_accuracy": 0.8730073302984238,
162
- "num_tokens": 15199243.0,
163
- "step": 140,
164
- "teacher_ce_loss": 2.522739
165
- },
166
- {
167
- "alignment_loss": 0.773578,
168
- "epoch": 0.07796511061300068,
169
- "grad_norm": 432.6204833984375,
170
- "learning_rate": 9.63783220427306e-06,
171
- "loss": 40.0022,
172
- "mean_token_accuracy": 0.8764732483774423,
173
- "num_tokens": 16291846.0,
174
- "step": 150,
175
- "teacher_ce_loss": 3.131191
176
- },
177
- {
178
- "alignment_loss": 0.784256,
179
- "epoch": 0.0831627846538674,
180
- "grad_norm": 454.5448913574219,
181
- "learning_rate": 9.611776967170403e-06,
182
- "loss": 42.8142,
183
- "mean_token_accuracy": 0.8686822090297938,
184
- "num_tokens": 17401156.0,
185
- "step": 160,
186
- "teacher_ce_loss": 2.710548
187
- },
188
- {
189
- "alignment_loss": 0.789857,
190
- "epoch": 0.0883604586947341,
191
- "grad_norm": 425.8789367675781,
192
- "learning_rate": 9.585721730067744e-06,
193
- "loss": 38.5832,
194
- "mean_token_accuracy": 0.8779909644275904,
195
- "num_tokens": 18515564.0,
196
- "step": 170,
197
- "teacher_ce_loss": 2.458202
198
- },
199
- {
200
- "alignment_loss": 0.785238,
201
- "epoch": 0.09355813273560082,
202
- "grad_norm": 455.0224914550781,
203
- "learning_rate": 9.559666492965087e-06,
204
- "loss": 43.4729,
205
- "mean_token_accuracy": 0.8689654342830181,
206
- "num_tokens": 19585633.0,
207
- "step": 180,
208
- "teacher_ce_loss": 2.883032
209
- },
210
- {
211
- "alignment_loss": 0.735123,
212
- "epoch": 0.09875580677646753,
213
- "grad_norm": 447.3720397949219,
214
- "learning_rate": 9.533611255862429e-06,
215
- "loss": 42.5676,
216
- "mean_token_accuracy": 0.8711532264947891,
217
- "num_tokens": 20652757.0,
218
- "step": 190,
219
- "teacher_ce_loss": 2.600116
220
- },
221
- {
222
- "alignment_loss": 0.760273,
223
- "epoch": 0.10395348081733424,
224
- "grad_norm": 486.74310302734375,
225
- "learning_rate": 9.507556018759772e-06,
226
- "loss": 41.0259,
227
- "mean_token_accuracy": 0.8750267956405878,
228
- "num_tokens": 21733652.0,
229
- "step": 200,
230
- "teacher_ce_loss": 2.561102
231
- },
232
- {
233
- "alignment_loss": 0.753341,
234
- "epoch": 0.10915115485820095,
235
- "grad_norm": 464.11993408203125,
236
- "learning_rate": 9.481500781657114e-06,
237
- "loss": 40.563,
238
- "mean_token_accuracy": 0.8767422955483198,
239
- "num_tokens": 22813402.0,
240
- "step": 210,
241
- "teacher_ce_loss": 2.690006
242
- },
243
- {
244
- "alignment_loss": 0.747512,
245
- "epoch": 0.11434882889906767,
246
- "grad_norm": 468.8255920410156,
247
- "learning_rate": 9.455445544554455e-06,
248
- "loss": 40.1685,
249
- "mean_token_accuracy": 0.8748798806220293,
250
- "num_tokens": 23921954.0,
251
- "step": 220,
252
- "teacher_ce_loss": 2.622118
253
- },
254
- {
255
- "alignment_loss": 0.753239,
256
- "epoch": 0.11954650293993438,
257
- "grad_norm": 403.46112060546875,
258
- "learning_rate": 9.429390307451798e-06,
259
- "loss": 42.2723,
260
- "mean_token_accuracy": 0.867292708531022,
261
- "num_tokens": 25026559.0,
262
- "step": 230,
263
- "teacher_ce_loss": 2.635886
264
- },
265
- {
266
- "alignment_loss": 0.745158,
267
- "epoch": 0.12474417698080109,
268
- "grad_norm": 474.7544860839844,
269
- "learning_rate": 9.403335070349142e-06,
270
- "loss": 41.1344,
271
- "mean_token_accuracy": 0.8710437923669815,
272
- "num_tokens": 26115485.0,
273
- "step": 240,
274
- "teacher_ce_loss": 2.360191
275
- },
276
- {
277
- "alignment_loss": 0.727475,
278
- "epoch": 0.1299418510216678,
279
- "grad_norm": 643.1618041992188,
280
- "learning_rate": 9.377279833246483e-06,
281
- "loss": 40.2501,
282
- "mean_token_accuracy": 0.8739985305815935,
283
- "num_tokens": 27193167.0,
284
- "step": 250,
285
- "teacher_ce_loss": 2.778879
286
- },
287
- {
288
- "alignment_loss": 0.734025,
289
- "epoch": 0.13513952506253452,
290
- "grad_norm": 401.61456298828125,
291
- "learning_rate": 9.351224596143825e-06,
292
- "loss": 41.9343,
293
- "mean_token_accuracy": 0.8782579999417066,
294
- "num_tokens": 28281924.0,
295
- "step": 260,
296
- "teacher_ce_loss": 2.480961
297
- },
298
- {
299
- "alignment_loss": 0.714433,
300
- "epoch": 0.14033719910340123,
301
- "grad_norm": 395.68505859375,
302
- "learning_rate": 9.325169359041168e-06,
303
- "loss": 39.6858,
304
- "mean_token_accuracy": 0.8774038635194301,
305
- "num_tokens": 29371146.0,
306
- "step": 270,
307
- "teacher_ce_loss": 2.480913
308
- },
309
- {
310
- "alignment_loss": 0.703511,
311
- "epoch": 0.14553487314426794,
312
- "grad_norm": 456.821533203125,
313
- "learning_rate": 9.299114121938511e-06,
314
- "loss": 40.3423,
315
- "mean_token_accuracy": 0.8739231277257204,
316
- "num_tokens": 30466570.0,
317
- "step": 280,
318
- "teacher_ce_loss": 2.296391
319
- },
320
- {
321
- "alignment_loss": 0.71004,
322
- "epoch": 0.15073254718513465,
323
- "grad_norm": 454.12408447265625,
324
- "learning_rate": 9.273058884835853e-06,
325
- "loss": 39.2462,
326
- "mean_token_accuracy": 0.8805094465613366,
327
- "num_tokens": 31555682.0,
328
- "step": 290,
329
- "teacher_ce_loss": 2.335335
330
- },
331
- {
332
- "alignment_loss": 0.686631,
333
- "epoch": 0.15593022122600136,
334
- "grad_norm": 427.1668701171875,
335
- "learning_rate": 9.247003647733194e-06,
336
- "loss": 40.4985,
337
- "mean_token_accuracy": 0.8747703462839127,
338
- "num_tokens": 32632292.0,
339
- "step": 300,
340
- "teacher_ce_loss": 2.695382
341
- },
342
- {
343
- "alignment_loss": 0.698419,
344
- "epoch": 0.16112789526686808,
345
- "grad_norm": 399.4712829589844,
346
- "learning_rate": 9.220948410630537e-06,
347
- "loss": 39.4977,
348
- "mean_token_accuracy": 0.8769657868891955,
349
- "num_tokens": 33700384.0,
350
- "step": 310,
351
- "teacher_ce_loss": 2.530473
352
- },
353
- {
354
- "alignment_loss": 0.686316,
355
- "epoch": 0.1663255693077348,
356
- "grad_norm": 458.28125,
357
- "learning_rate": 9.19489317352788e-06,
358
- "loss": 40.7732,
359
- "mean_token_accuracy": 0.8730976637452841,
360
- "num_tokens": 34792358.0,
361
- "step": 320,
362
- "teacher_ce_loss": 2.44503
363
- },
364
- {
365
- "alignment_loss": 0.663645,
366
- "epoch": 0.1715232433486015,
367
- "grad_norm": 492.2942810058594,
368
- "learning_rate": 9.168837936425222e-06,
369
- "loss": 40.9792,
370
- "mean_token_accuracy": 0.871804180368781,
371
- "num_tokens": 35869379.0,
372
- "step": 330,
373
- "teacher_ce_loss": 2.677346
374
- },
375
- {
376
- "alignment_loss": 0.668165,
377
- "epoch": 0.1767209173894682,
378
- "grad_norm": 486.4925537109375,
379
- "learning_rate": 9.142782699322564e-06,
380
- "loss": 37.4871,
381
- "mean_token_accuracy": 0.8835177160799503,
382
- "num_tokens": 36961278.0,
383
- "step": 340,
384
- "teacher_ce_loss": 2.241284
385
- },
386
- {
387
- "alignment_loss": 0.664259,
388
- "epoch": 0.18191859143033492,
389
- "grad_norm": 401.2726135253906,
390
- "learning_rate": 9.116727462219907e-06,
391
- "loss": 40.2021,
392
- "mean_token_accuracy": 0.878755921125412,
393
- "num_tokens": 37997477.0,
394
- "step": 350,
395
- "teacher_ce_loss": 2.493072
396
- },
397
- {
398
- "alignment_loss": 0.635681,
399
- "epoch": 0.18711626547120164,
400
- "grad_norm": 482.34722900390625,
401
- "learning_rate": 9.09067222511725e-06,
402
- "loss": 39.7491,
403
- "mean_token_accuracy": 0.8715844135731459,
404
- "num_tokens": 39107550.0,
405
- "step": 360,
406
- "teacher_ce_loss": 2.646277
407
- },
408
- {
409
- "alignment_loss": 0.663796,
410
- "epoch": 0.19231393951206835,
411
- "grad_norm": 416.504638671875,
412
- "learning_rate": 9.064616988014592e-06,
413
- "loss": 37.9154,
414
- "mean_token_accuracy": 0.8845120508223772,
415
- "num_tokens": 40186985.0,
416
- "step": 370,
417
- "teacher_ce_loss": 2.248134
418
- },
419
- {
420
- "alignment_loss": 0.645071,
421
- "epoch": 0.19751161355293506,
422
- "grad_norm": 429.7480773925781,
423
- "learning_rate": 9.038561750911933e-06,
424
- "loss": 38.9221,
425
- "mean_token_accuracy": 0.8787473827600479,
426
- "num_tokens": 41294886.0,
427
- "step": 380,
428
- "teacher_ce_loss": 2.640242
429
- },
430
- {
431
- "alignment_loss": 0.634285,
432
- "epoch": 0.20270928759380177,
433
- "grad_norm": 397.76898193359375,
434
- "learning_rate": 9.012506513809276e-06,
435
- "loss": 38.3401,
436
- "mean_token_accuracy": 0.8707687258720398,
437
- "num_tokens": 42402780.0,
438
- "step": 390,
439
- "teacher_ce_loss": 2.452602
440
- },
441
- {
442
- "alignment_loss": 0.644811,
443
- "epoch": 0.20790696163466849,
444
- "grad_norm": 555.0195922851562,
445
- "learning_rate": 8.98645127670662e-06,
446
- "loss": 38.127,
447
- "mean_token_accuracy": 0.8795747645199299,
448
- "num_tokens": 43488761.0,
449
- "step": 400,
450
- "teacher_ce_loss": 2.41793
451
- },
452
- {
453
- "alignment_loss": 0.628519,
454
- "epoch": 0.2131046356755352,
455
- "grad_norm": 412.0311279296875,
456
- "learning_rate": 8.960396039603961e-06,
457
- "loss": 38.585,
458
- "mean_token_accuracy": 0.8811177968978882,
459
- "num_tokens": 44563824.0,
460
- "step": 410,
461
- "teacher_ce_loss": 2.548315
462
- },
463
- {
464
- "alignment_loss": 0.606575,
465
- "epoch": 0.2183023097164019,
466
- "grad_norm": 418.9155578613281,
467
- "learning_rate": 8.934340802501303e-06,
468
- "loss": 37.2091,
469
- "mean_token_accuracy": 0.8801510404795408,
470
- "num_tokens": 45648068.0,
471
- "step": 420,
472
- "teacher_ce_loss": 2.284262
473
- },
474
- {
475
- "alignment_loss": 0.590465,
476
- "epoch": 0.22349998375726862,
477
- "grad_norm": 389.65313720703125,
478
- "learning_rate": 8.908285565398646e-06,
479
- "loss": 37.5798,
480
- "mean_token_accuracy": 0.8835781816393137,
481
- "num_tokens": 46734913.0,
482
- "step": 430,
483
- "teacher_ce_loss": 2.463078
484
- },
485
- {
486
- "alignment_loss": 0.61147,
487
- "epoch": 0.22869765779813533,
488
- "grad_norm": 430.8939514160156,
489
- "learning_rate": 8.882230328295989e-06,
490
- "loss": 38.0461,
491
- "mean_token_accuracy": 0.8793204519897699,
492
- "num_tokens": 47826491.0,
493
- "step": 440,
494
- "teacher_ce_loss": 2.524572
495
- },
496
- {
497
- "alignment_loss": 0.597793,
498
- "epoch": 0.23389533183900205,
499
- "grad_norm": 486.86968994140625,
500
- "learning_rate": 8.85617509119333e-06,
501
- "loss": 37.3776,
502
- "mean_token_accuracy": 0.8813808042556047,
503
- "num_tokens": 48880880.0,
504
- "step": 450,
505
- "teacher_ce_loss": 2.364488
506
- },
507
- {
508
- "alignment_loss": 0.609329,
509
- "epoch": 0.23909300587986876,
510
- "grad_norm": 417.3847961425781,
511
- "learning_rate": 8.830119854090674e-06,
512
- "loss": 37.8343,
513
- "mean_token_accuracy": 0.8755738019943238,
514
- "num_tokens": 49980318.0,
515
- "step": 460,
516
- "teacher_ce_loss": 2.416585
517
- },
518
- {
519
- "alignment_loss": 0.604474,
520
- "epoch": 0.24429067992073547,
521
- "grad_norm": 406.4049072265625,
522
- "learning_rate": 8.804064616988015e-06,
523
- "loss": 36.6321,
524
- "mean_token_accuracy": 0.8823226012289525,
525
- "num_tokens": 51098775.0,
526
- "step": 470,
527
- "teacher_ce_loss": 2.540241
528
- },
529
- {
530
- "alignment_loss": 0.595836,
531
- "epoch": 0.24948835396160218,
532
- "grad_norm": 467.265869140625,
533
- "learning_rate": 8.778009379885357e-06,
534
- "loss": 36.7995,
535
- "mean_token_accuracy": 0.8795299742370843,
536
- "num_tokens": 52217205.0,
537
- "step": 480,
538
- "teacher_ce_loss": 2.563789
539
- },
540
- {
541
- "alignment_loss": 0.578717,
542
- "epoch": 0.2546860280024689,
543
- "grad_norm": 373.62457275390625,
544
- "learning_rate": 8.7519541427827e-06,
545
- "loss": 38.4347,
546
- "mean_token_accuracy": 0.8753514669835567,
547
- "num_tokens": 53274010.0,
548
- "step": 490,
549
- "teacher_ce_loss": 2.481531
550
- },
551
- {
552
- "alignment_loss": 0.575263,
553
- "epoch": 0.2598837020433356,
554
- "grad_norm": 433.00665283203125,
555
- "learning_rate": 8.725898905680043e-06,
556
- "loss": 37.3274,
557
- "mean_token_accuracy": 0.8773705784231425,
558
- "num_tokens": 54366960.0,
559
- "step": 500,
560
- "teacher_ce_loss": 2.842024
561
- },
562
- {
563
- "alignment_loss": 0.587993,
564
- "epoch": 0.26508137608420235,
565
- "grad_norm": 353.8840637207031,
566
- "learning_rate": 8.699843668577385e-06,
567
- "loss": 38.3482,
568
- "mean_token_accuracy": 0.8801531791687012,
569
- "num_tokens": 55439538.0,
570
- "step": 510,
571
- "teacher_ce_loss": 2.274705
572
- },
573
- {
574
- "alignment_loss": 0.588782,
575
- "epoch": 0.27027905012506903,
576
- "grad_norm": 333.64227294921875,
577
- "learning_rate": 8.673788431474726e-06,
578
- "loss": 37.1398,
579
- "mean_token_accuracy": 0.8774184010922909,
580
- "num_tokens": 56526079.0,
581
- "step": 520,
582
- "teacher_ce_loss": 2.613483
583
- },
584
- {
585
- "alignment_loss": 0.554341,
586
- "epoch": 0.27547672416593577,
587
- "grad_norm": 465.486328125,
588
- "learning_rate": 8.64773319437207e-06,
589
- "loss": 36.1345,
590
- "mean_token_accuracy": 0.8829835120588541,
591
- "num_tokens": 57597194.0,
592
- "step": 530,
593
- "teacher_ce_loss": 2.127812
594
- },
595
- {
596
- "alignment_loss": 0.58787,
597
- "epoch": 0.28067439820680246,
598
- "grad_norm": 429.91424560546875,
599
- "learning_rate": 8.621677957269413e-06,
600
- "loss": 37.8734,
601
- "mean_token_accuracy": 0.8773438431322574,
602
- "num_tokens": 58659824.0,
603
- "step": 540,
604
- "teacher_ce_loss": 1.918437
605
- },
606
- {
607
- "alignment_loss": 0.575647,
608
- "epoch": 0.2858720722476692,
609
- "grad_norm": 349.5917663574219,
610
- "learning_rate": 8.595622720166754e-06,
611
- "loss": 38.1177,
612
- "mean_token_accuracy": 0.8757255170494318,
613
- "num_tokens": 59763185.0,
614
- "step": 550,
615
- "teacher_ce_loss": 2.450488
616
- },
617
- {
618
- "alignment_loss": 0.564791,
619
- "epoch": 0.2910697462885359,
620
- "grad_norm": 344.8094787597656,
621
- "learning_rate": 8.569567483064096e-06,
622
- "loss": 36.5853,
623
- "mean_token_accuracy": 0.8832464355975389,
624
- "num_tokens": 60859991.0,
625
- "step": 560,
626
- "teacher_ce_loss": 2.646985
627
- },
628
- {
629
- "alignment_loss": 0.592017,
630
- "epoch": 0.2962674203294026,
631
- "grad_norm": 440.2308349609375,
632
- "learning_rate": 8.543512245961439e-06,
633
- "loss": 35.0455,
634
- "mean_token_accuracy": 0.8869358003139496,
635
- "num_tokens": 61953471.0,
636
- "step": 570,
637
- "teacher_ce_loss": 2.230956
638
- },
639
- {
640
- "alignment_loss": 0.570755,
641
- "epoch": 0.3014650943702693,
642
- "grad_norm": 376.47857666015625,
643
- "learning_rate": 8.517457008858782e-06,
644
- "loss": 37.0786,
645
- "mean_token_accuracy": 0.8792325057089329,
646
- "num_tokens": 63040667.0,
647
- "step": 580,
648
- "teacher_ce_loss": 2.573731
649
- },
650
- {
651
- "alignment_loss": 0.558999,
652
- "epoch": 0.30666276841113604,
653
- "grad_norm": 355.3178405761719,
654
- "learning_rate": 8.491401771756124e-06,
655
- "loss": 34.1199,
656
- "mean_token_accuracy": 0.8901216987520456,
657
- "num_tokens": 64125721.0,
658
- "step": 590,
659
- "teacher_ce_loss": 2.449002
660
- },
661
- {
662
- "alignment_loss": 0.544031,
663
- "epoch": 0.31186044245200273,
664
- "grad_norm": 365.6088562011719,
665
- "learning_rate": 8.465346534653465e-06,
666
- "loss": 35.5018,
667
- "mean_token_accuracy": 0.8875447463244199,
668
- "num_tokens": 65229128.0,
669
- "step": 600,
670
- "teacher_ce_loss": 2.58022
671
- },
672
- {
673
- "alignment_loss": 0.561479,
674
- "epoch": 0.31705811649286947,
675
- "grad_norm": 371.2009582519531,
676
- "learning_rate": 8.439291297550808e-06,
677
- "loss": 36.8428,
678
- "mean_token_accuracy": 0.8801365926861763,
679
- "num_tokens": 66330522.0,
680
- "step": 610,
681
- "teacher_ce_loss": 2.131811
682
- },
683
- {
684
- "alignment_loss": 0.567115,
685
- "epoch": 0.32225579053373615,
686
- "grad_norm": 371.50482177734375,
687
- "learning_rate": 8.413236060448152e-06,
688
- "loss": 35.886,
689
- "mean_token_accuracy": 0.8856256190687418,
690
- "num_tokens": 67401628.0,
691
- "step": 620,
692
- "teacher_ce_loss": 2.546763
693
- },
694
- {
695
- "alignment_loss": 0.56429,
696
- "epoch": 0.3274534645746029,
697
- "grad_norm": 368.6912841796875,
698
- "learning_rate": 8.387180823345493e-06,
699
- "loss": 35.4131,
700
- "mean_token_accuracy": 0.8811270847916604,
701
- "num_tokens": 68488661.0,
702
- "step": 630,
703
- "teacher_ce_loss": 2.076486
704
- },
705
- {
706
- "alignment_loss": 0.54927,
707
- "epoch": 0.3326511386154696,
708
- "grad_norm": 348.7677001953125,
709
- "learning_rate": 8.361125586242835e-06,
710
- "loss": 37.1297,
711
- "mean_token_accuracy": 0.8735985334962606,
712
- "num_tokens": 69565168.0,
713
- "step": 640,
714
- "teacher_ce_loss": 2.419532
715
- },
716
- {
717
- "alignment_loss": 0.565124,
718
- "epoch": 0.3378488126563363,
719
- "grad_norm": 315.0074157714844,
720
- "learning_rate": 8.335070349140178e-06,
721
- "loss": 37.476,
722
- "mean_token_accuracy": 0.8793271500617266,
723
- "num_tokens": 70658047.0,
724
- "step": 650,
725
- "teacher_ce_loss": 2.50526
726
- },
727
- {
728
- "alignment_loss": 0.56105,
729
- "epoch": 0.343046486697203,
730
- "grad_norm": 370.9797058105469,
731
- "learning_rate": 8.309015112037521e-06,
732
- "loss": 36.4934,
733
- "mean_token_accuracy": 0.8765981134027243,
734
- "num_tokens": 71751817.0,
735
- "step": 660,
736
- "teacher_ce_loss": 2.628102
737
- },
738
- {
739
- "alignment_loss": 0.558229,
740
- "epoch": 0.34824416073806974,
741
- "grad_norm": 407.77618408203125,
742
- "learning_rate": 8.282959874934863e-06,
743
- "loss": 35.6226,
744
- "mean_token_accuracy": 0.8887185551226139,
745
- "num_tokens": 72832097.0,
746
- "step": 670,
747
- "teacher_ce_loss": 2.322924
748
- },
749
- {
750
- "alignment_loss": 0.563812,
751
- "epoch": 0.3534418347789364,
752
- "grad_norm": 374.8846435546875,
753
- "learning_rate": 8.256904637832204e-06,
754
- "loss": 35.7817,
755
- "mean_token_accuracy": 0.8814776569604874,
756
- "num_tokens": 73901429.0,
757
- "step": 680,
758
- "teacher_ce_loss": 2.11159
759
- },
760
- {
761
- "alignment_loss": 0.562092,
762
- "epoch": 0.35863950881980317,
763
- "grad_norm": 374.0208740234375,
764
- "learning_rate": 8.230849400729547e-06,
765
- "loss": 37.8348,
766
- "mean_token_accuracy": 0.8765548631548882,
767
- "num_tokens": 74943337.0,
768
- "step": 690,
769
- "teacher_ce_loss": 2.035791
770
- },
771
- {
772
- "alignment_loss": 0.548845,
773
- "epoch": 0.36383718286066985,
774
- "grad_norm": 334.5244445800781,
775
- "learning_rate": 8.20479416362689e-06,
776
- "loss": 36.0082,
777
- "mean_token_accuracy": 0.8834207616746426,
778
- "num_tokens": 76033070.0,
779
- "step": 700,
780
- "teacher_ce_loss": 1.906485
781
- },
782
- {
783
- "alignment_loss": 0.557486,
784
- "epoch": 0.3690348569015366,
785
- "grad_norm": 319.5161437988281,
786
- "learning_rate": 8.178738926524232e-06,
787
- "loss": 34.7519,
788
- "mean_token_accuracy": 0.8918090540915727,
789
- "num_tokens": 77127837.0,
790
- "step": 710,
791
- "teacher_ce_loss": 2.22631
792
- },
793
- {
794
- "alignment_loss": 0.544557,
795
- "epoch": 0.3742325309424033,
796
- "grad_norm": 416.67138671875,
797
- "learning_rate": 8.152683689421574e-06,
798
- "loss": 36.0742,
799
- "mean_token_accuracy": 0.8771771170198918,
800
- "num_tokens": 78213233.0,
801
- "step": 720,
802
- "teacher_ce_loss": 2.096081
803
- },
804
- {
805
- "alignment_loss": 0.542186,
806
- "epoch": 0.37943020498327,
807
- "grad_norm": 366.275390625,
808
- "learning_rate": 8.126628452318917e-06,
809
- "loss": 34.6399,
810
- "mean_token_accuracy": 0.8871560543775558,
811
- "num_tokens": 79278369.0,
812
- "step": 730,
813
- "teacher_ce_loss": 2.093614
814
- },
815
- {
816
- "alignment_loss": 0.55917,
817
- "epoch": 0.3846278790241367,
818
- "grad_norm": 343.9064025878906,
819
- "learning_rate": 8.100573215216258e-06,
820
- "loss": 35.9346,
821
- "mean_token_accuracy": 0.8843456242233515,
822
- "num_tokens": 80371795.0,
823
- "step": 740,
824
- "teacher_ce_loss": 2.285846
825
- },
826
- {
827
- "alignment_loss": 0.566925,
828
- "epoch": 0.38982555306500344,
829
- "grad_norm": 348.0086364746094,
830
- "learning_rate": 8.074517978113602e-06,
831
- "loss": 39.5565,
832
- "mean_token_accuracy": 0.864898469671607,
833
- "num_tokens": 81448889.0,
834
- "step": 750,
835
- "teacher_ce_loss": 2.940448
836
- },
837
- {
838
- "alignment_loss": 0.556261,
839
- "epoch": 0.3950232271058701,
840
- "grad_norm": 386.3326416015625,
841
- "learning_rate": 8.048462741010943e-06,
842
- "loss": 34.5507,
843
- "mean_token_accuracy": 0.885974370315671,
844
- "num_tokens": 82529410.0,
845
- "step": 760,
846
- "teacher_ce_loss": 2.123303
847
- },
848
- {
849
- "alignment_loss": 0.552616,
850
- "epoch": 0.40022090114673686,
851
- "grad_norm": 357.1813049316406,
852
- "learning_rate": 8.022407503908286e-06,
853
- "loss": 35.0587,
854
- "mean_token_accuracy": 0.8836169846355915,
855
- "num_tokens": 83621744.0,
856
- "step": 770,
857
- "teacher_ce_loss": 2.308799
858
- },
859
- {
860
- "alignment_loss": 0.554798,
861
- "epoch": 0.40541857518760355,
862
- "grad_norm": 363.66546630859375,
863
- "learning_rate": 7.996352266805628e-06,
864
- "loss": 38.2547,
865
- "mean_token_accuracy": 0.8708455882966518,
866
- "num_tokens": 84688203.0,
867
- "step": 780,
868
- "teacher_ce_loss": 2.409154
869
- },
870
- {
871
- "alignment_loss": 0.540404,
872
- "epoch": 0.4106162492284703,
873
- "grad_norm": 392.4259948730469,
874
- "learning_rate": 7.970297029702971e-06,
875
- "loss": 34.9898,
876
- "mean_token_accuracy": 0.8874543648213148,
877
- "num_tokens": 85778396.0,
878
- "step": 790,
879
- "teacher_ce_loss": 2.510973
880
- },
881
- {
882
- "alignment_loss": 0.525033,
883
- "epoch": 0.41581392326933697,
884
- "grad_norm": 338.7568359375,
885
- "learning_rate": 7.944241792600313e-06,
886
- "loss": 36.0635,
887
- "mean_token_accuracy": 0.8779846042394638,
888
- "num_tokens": 86851299.0,
889
- "step": 800,
890
- "teacher_ce_loss": 2.082863
891
- },
892
- {
893
- "alignment_loss": 0.537521,
894
- "epoch": 0.4210115973102037,
895
- "grad_norm": 349.5997009277344,
896
- "learning_rate": 7.918186555497656e-06,
897
- "loss": 35.6515,
898
- "mean_token_accuracy": 0.878065213188529,
899
- "num_tokens": 87923725.0,
900
- "step": 810,
901
- "teacher_ce_loss": 2.233534
902
- },
903
- {
904
- "alignment_loss": 0.529449,
905
- "epoch": 0.4262092713510704,
906
- "grad_norm": 366.2076416015625,
907
- "learning_rate": 7.892131318394997e-06,
908
- "loss": 33.9753,
909
- "mean_token_accuracy": 0.8886250294744968,
910
- "num_tokens": 89012030.0,
911
- "step": 820,
912
- "teacher_ce_loss": 2.235799
913
- },
914
- {
915
- "alignment_loss": 0.540571,
916
- "epoch": 0.43140694539193714,
917
- "grad_norm": 356.45220947265625,
918
- "learning_rate": 7.86607608129234e-06,
919
- "loss": 34.3357,
920
- "mean_token_accuracy": 0.880321879312396,
921
- "num_tokens": 90082610.0,
922
- "step": 830,
923
- "teacher_ce_loss": 2.216298
924
- },
925
- {
926
- "alignment_loss": 0.569118,
927
- "epoch": 0.4366046194328038,
928
- "grad_norm": 324.5358581542969,
929
- "learning_rate": 7.840020844189684e-06,
930
- "loss": 36.1544,
931
- "mean_token_accuracy": 0.8789420172572135,
932
- "num_tokens": 91178464.0,
933
- "step": 840,
934
- "teacher_ce_loss": 2.117879
935
- },
936
- {
937
- "alignment_loss": 0.557968,
938
- "epoch": 0.44180229347367056,
939
- "grad_norm": 349.2758483886719,
940
- "learning_rate": 7.813965607087025e-06,
941
- "loss": 34.2247,
942
- "mean_token_accuracy": 0.8846652548760175,
943
- "num_tokens": 92249627.0,
944
- "step": 850,
945
- "teacher_ce_loss": 2.37176
946
- },
947
- {
948
- "alignment_loss": 0.528144,
949
- "epoch": 0.44699996751453724,
950
- "grad_norm": 402.4473876953125,
951
- "learning_rate": 7.787910369984367e-06,
952
- "loss": 35.8188,
953
- "mean_token_accuracy": 0.8837881837040186,
954
- "num_tokens": 93332295.0,
955
- "step": 860,
956
- "teacher_ce_loss": 2.205281
957
- },
958
- {
959
- "alignment_loss": 0.549412,
960
- "epoch": 0.452197641555404,
961
- "grad_norm": 356.1290283203125,
962
- "learning_rate": 7.76185513288171e-06,
963
- "loss": 34.1296,
964
- "mean_token_accuracy": 0.879589206725359,
965
- "num_tokens": 94435236.0,
966
- "step": 870,
967
- "teacher_ce_loss": 2.50075
968
- },
969
- {
970
- "alignment_loss": 0.566606,
971
- "epoch": 0.45739531559627067,
972
- "grad_norm": 377.2438659667969,
973
- "learning_rate": 7.735799895779053e-06,
974
- "loss": 34.9863,
975
- "mean_token_accuracy": 0.876738016679883,
976
- "num_tokens": 95510073.0,
977
- "step": 880,
978
- "teacher_ce_loss": 2.361031
979
- },
980
- {
981
- "alignment_loss": 0.553246,
982
- "epoch": 0.4625929896371374,
983
- "grad_norm": 341.814208984375,
984
- "learning_rate": 7.709744658676395e-06,
985
- "loss": 32.0134,
986
- "mean_token_accuracy": 0.8967050477862358,
987
- "num_tokens": 96615597.0,
988
- "step": 890,
989
- "teacher_ce_loss": 2.078417
990
- },
991
- {
992
- "alignment_loss": 0.558538,
993
- "epoch": 0.4677906636780041,
994
- "grad_norm": 333.966552734375,
995
- "learning_rate": 7.683689421573736e-06,
996
- "loss": 34.7242,
997
- "mean_token_accuracy": 0.8789819356054067,
998
- "num_tokens": 97719258.0,
999
- "step": 900,
1000
- "teacher_ce_loss": 3.110485
1001
- },
1002
- {
1003
- "alignment_loss": 0.54561,
1004
- "epoch": 0.47298833771887083,
1005
- "grad_norm": 381.2041015625,
1006
- "learning_rate": 7.65763418447108e-06,
1007
- "loss": 29.1169,
1008
- "mean_token_accuracy": 0.8953132223337888,
1009
- "num_tokens": 98804865.0,
1010
- "step": 910,
1011
- "teacher_ce_loss": 1.670265
1012
- },
1013
- {
1014
- "alignment_loss": 0.54926,
1015
- "epoch": 0.4781860117597375,
1016
- "grad_norm": 475.4048767089844,
1017
- "learning_rate": 7.631578947368423e-06,
1018
- "loss": 35.1457,
1019
- "mean_token_accuracy": 0.8785753037780524,
1020
- "num_tokens": 99907921.0,
1021
- "step": 920,
1022
- "teacher_ce_loss": 1.938184
1023
- },
1024
- {
1025
- "alignment_loss": 0.5388,
1026
- "epoch": 0.48338368580060426,
1027
- "grad_norm": 340.2969970703125,
1028
- "learning_rate": 7.605523710265764e-06,
1029
- "loss": 34.7152,
1030
- "mean_token_accuracy": 0.882407446205616,
1031
- "num_tokens": 100974965.0,
1032
- "step": 930,
1033
- "teacher_ce_loss": 2.444891
1034
- },
1035
- {
1036
- "alignment_loss": 0.535554,
1037
- "epoch": 0.48858135984147094,
1038
- "grad_norm": 393.9867858886719,
1039
- "learning_rate": 7.579468473163107e-06,
1040
- "loss": 34.8008,
1041
- "mean_token_accuracy": 0.8857435449957848,
1042
- "num_tokens": 102044019.0,
1043
- "step": 940,
1044
- "teacher_ce_loss": 2.369765
1045
- },
1046
- {
1047
- "alignment_loss": 0.539926,
1048
- "epoch": 0.4937790338823377,
1049
- "grad_norm": 369.8341064453125,
1050
- "learning_rate": 7.553413236060448e-06,
1051
- "loss": 34.4707,
1052
- "mean_token_accuracy": 0.8845079831779004,
1053
- "num_tokens": 103124685.0,
1054
- "step": 950,
1055
- "teacher_ce_loss": 2.564217
1056
- },
1057
- {
1058
- "alignment_loss": 0.543687,
1059
- "epoch": 0.49897670792320437,
1060
- "grad_norm": 361.390625,
1061
- "learning_rate": 7.527357998957791e-06,
1062
- "loss": 34.6248,
1063
- "mean_token_accuracy": 0.8883094284683466,
1064
- "num_tokens": 104225407.0,
1065
- "step": 960,
1066
- "teacher_ce_loss": 1.582248
1067
- },
1068
- {
1069
- "alignment_loss": 0.548604,
1070
- "epoch": 0.504174381964071,
1071
- "grad_norm": 385.1660461425781,
1072
- "learning_rate": 7.501302761855134e-06,
1073
- "loss": 33.6384,
1074
- "mean_token_accuracy": 0.8866939105093479,
1075
- "num_tokens": 105289284.0,
1076
- "step": 970,
1077
- "teacher_ce_loss": 2.256051
1078
- },
1079
- {
1080
- "alignment_loss": 0.528196,
1081
- "epoch": 0.5093720560049378,
1082
- "grad_norm": 364.2571105957031,
1083
- "learning_rate": 7.475247524752476e-06,
1084
- "loss": 34.621,
1085
- "mean_token_accuracy": 0.8826428797096014,
1086
- "num_tokens": 106362820.0,
1087
- "step": 980,
1088
- "teacher_ce_loss": 2.33378
1089
- },
1090
- {
1091
- "alignment_loss": 0.533318,
1092
- "epoch": 0.5145697300458045,
1093
- "grad_norm": 357.0517883300781,
1094
- "learning_rate": 7.449192287649818e-06,
1095
- "loss": 32.3759,
1096
- "mean_token_accuracy": 0.8855575665831565,
1097
- "num_tokens": 107467590.0,
1098
- "step": 990,
1099
- "teacher_ce_loss": 2.30613
1100
- },
1101
- {
1102
- "alignment_loss": 0.53744,
1103
- "epoch": 0.5197674040866712,
1104
- "grad_norm": 304.0802307128906,
1105
- "learning_rate": 7.42313705054716e-06,
1106
- "loss": 35.2977,
1107
- "mean_token_accuracy": 0.8794034343212843,
1108
- "num_tokens": 108550005.0,
1109
- "step": 1000,
1110
- "teacher_ce_loss": 2.076965
1111
- }
1112
- ],
1113
- "logging_steps": 10,
1114
- "max_steps": 3848,
1115
- "num_input_tokens_seen": 0,
1116
- "num_train_epochs": 2,
1117
- "save_steps": 50,
1118
- "stateful_callbacks": {
1119
- "TrainerControl": {
1120
- "args": {
1121
- "should_epoch_stop": false,
1122
- "should_evaluate": false,
1123
- "should_log": false,
1124
- "should_save": true,
1125
- "should_training_stop": false
1126
- },
1127
- "attributes": {}
1128
- }
1129
- },
1130
- "total_flos": 5.044811883723358e+18,
1131
- "train_batch_size": 1,
1132
- "trial_name": null,
1133
- "trial_params": null
1134
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e861a7238207eaec94b085e7f17b60738ef5e4b9c421e456a6d3c521acae754
3
- size 10570
 
 
 
 
size-250k/checkpoint-1000/video_preprocessor_config.json DELETED
@@ -1,43 +0,0 @@
1
- {
2
- "crop_size": null,
3
- "data_format": "channels_first",
4
- "default_to_square": true,
5
- "device": null,
6
- "do_center_crop": null,
7
- "do_convert_rgb": true,
8
- "do_normalize": true,
9
- "do_pad": null,
10
- "do_rescale": true,
11
- "do_resize": true,
12
- "do_sample_frames": false,
13
- "fps": null,
14
- "image_mean": [
15
- 0.48145466,
16
- 0.4578275,
17
- 0.40821073
18
- ],
19
- "image_std": [
20
- 0.26862954,
21
- 0.26130258,
22
- 0.27577711
23
- ],
24
- "input_data_format": null,
25
- "max_frames": 768,
26
- "max_pixels": 12845056,
27
- "merge_size": 2,
28
- "min_frames": 4,
29
- "min_pixels": 3136,
30
- "num_frames": null,
31
- "patch_size": 14,
32
- "processor_class": "Qwen2_5_VLProcessor",
33
- "resample": 3,
34
- "rescale_factor": 0.00392156862745098,
35
- "size": {
36
- "longest_edge": 12845056,
37
- "shortest_edge": 3136
38
- },
39
- "size_divisor": null,
40
- "temporal_patch_size": 2,
41
- "video_metadata": null,
42
- "video_processor_type": "Qwen2VLVideoProcessor"
43
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-250k/checkpoint-1000/vocab.json DELETED
The diff for this file is too large to render. See raw diff
 
size-250k/checkpoint-1000/zero_to_fp32.py DELETED
@@ -1,760 +0,0 @@
1
- #!/usr/bin/env python
2
-
3
- # Copyright (c) Microsoft Corporation.
4
- # SPDX-License-Identifier: Apache-2.0
5
-
6
- # DeepSpeed Team
7
-
8
- # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
- # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
- # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
- # application.
12
- #
13
- # example:
14
- # python zero_to_fp32.py . output_dir/
15
- # or
16
- # python zero_to_fp32.py . output_dir/ --safe_serialization
17
-
18
- import argparse
19
- import torch
20
- import glob
21
- import math
22
- import os
23
- import re
24
- import gc
25
- import json
26
- import numpy as np
27
- from tqdm import tqdm
28
- from collections import OrderedDict
29
- from dataclasses import dataclass
30
-
31
- # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
- # DeepSpeed data structures it has to be available in the current python environment.
33
- from deepspeed.utils import logger
34
- from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
- FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
- FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
-
38
-
39
- @dataclass
40
- class zero_model_state:
41
- buffers: dict()
42
- param_shapes: dict()
43
- shared_params: list
44
- ds_version: int
45
- frozen_param_shapes: dict()
46
- frozen_param_fragments: dict()
47
-
48
-
49
- debug = 0
50
-
51
- # load to cpu
52
- device = torch.device('cpu')
53
-
54
-
55
- def atoi(text):
56
- return int(text) if text.isdigit() else text
57
-
58
-
59
- def natural_keys(text):
60
- '''
61
- alist.sort(key=natural_keys) sorts in human order
62
- http://nedbatchelder.com/blog/200712/human_sorting.html
63
- (See Toothy's implementation in the comments)
64
- '''
65
- return [atoi(c) for c in re.split(r'(\d+)', text)]
66
-
67
-
68
- def get_model_state_file(checkpoint_dir, zero_stage):
69
- if not os.path.isdir(checkpoint_dir):
70
- raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
-
72
- # there should be only one file
73
- if zero_stage <= 2:
74
- file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
- elif zero_stage == 3:
76
- file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
-
78
- if not os.path.exists(file):
79
- raise FileNotFoundError(f"can't find model states file at '{file}'")
80
-
81
- return file
82
-
83
-
84
- def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
- # XXX: need to test that this simple glob rule works for multi-node setup too
86
- ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
-
88
- if len(ckpt_files) == 0:
89
- raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
-
91
- return ckpt_files
92
-
93
-
94
- def get_optim_files(checkpoint_dir):
95
- return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
-
97
-
98
- def get_model_state_files(checkpoint_dir):
99
- return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
-
101
-
102
- def parse_model_states(files):
103
- zero_model_states = []
104
- for file in files:
105
- state_dict = torch.load(file, map_location=device, weights_only=False)
106
-
107
- if BUFFER_NAMES not in state_dict:
108
- raise ValueError(f"{file} is not a model state checkpoint")
109
- buffer_names = state_dict[BUFFER_NAMES]
110
- if debug:
111
- print("Found buffers:", buffer_names)
112
-
113
- # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
- buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
- param_shapes = state_dict[PARAM_SHAPES]
116
-
117
- # collect parameters that are included in param_shapes
118
- param_names = []
119
- for s in param_shapes:
120
- for name in s.keys():
121
- param_names.append(name)
122
-
123
- # update with frozen parameters
124
- frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
- if frozen_param_shapes is not None:
126
- if debug:
127
- print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
- param_names += list(frozen_param_shapes.keys())
129
-
130
- # handle shared params
131
- shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
-
133
- ds_version = state_dict.get(DS_VERSION, None)
134
-
135
- frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
-
137
- z_model_state = zero_model_state(buffers=buffers,
138
- param_shapes=param_shapes,
139
- shared_params=shared_params,
140
- ds_version=ds_version,
141
- frozen_param_shapes=frozen_param_shapes,
142
- frozen_param_fragments=frozen_param_fragments)
143
- zero_model_states.append(z_model_state)
144
-
145
- return zero_model_states
146
-
147
-
148
- def parse_optim_states(files, ds_checkpoint_dir):
149
- total_files = len(files)
150
- state_dicts = []
151
- for f in tqdm(files, desc='Loading checkpoint shards'):
152
- state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
- # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
- # and also handle the case where it was already removed by another helper script
155
- state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
- state_dicts.append(state_dict)
157
-
158
- if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
- raise ValueError(f"{files[0]} is not a zero checkpoint")
160
- zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
- world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
-
163
- # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
- # parameters can be different from data parallelism for non-expert parameters. So we can just
165
- # use the max of the partition_count to get the dp world_size.
166
-
167
- if type(world_size) is list:
168
- world_size = max(world_size)
169
-
170
- if world_size != total_files:
171
- raise ValueError(
172
- f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
- "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
- )
175
-
176
- # the groups are named differently in each stage
177
- if zero_stage <= 2:
178
- fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
- elif zero_stage == 3:
180
- fp32_groups_key = FP32_FLAT_GROUPS
181
- else:
182
- raise ValueError(f"unknown zero stage {zero_stage}")
183
-
184
- fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
- return zero_stage, world_size, fp32_flat_groups
186
-
187
-
188
- def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
- """
190
- Returns fp32 state_dict reconstructed from ds checkpoint
191
-
192
- Args:
193
- - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
-
195
- """
196
- print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
-
198
- optim_files = get_optim_files(ds_checkpoint_dir)
199
- zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
- print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
-
202
- model_files = get_model_state_files(ds_checkpoint_dir)
203
-
204
- zero_model_states = parse_model_states(model_files)
205
- print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
-
207
- if zero_stage <= 2:
208
- return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
- exclude_frozen_parameters)
210
- elif zero_stage == 3:
211
- return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
- exclude_frozen_parameters)
213
-
214
-
215
- def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
- if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
- return
218
-
219
- frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
- frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
-
222
- if debug:
223
- num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
- print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
-
226
- wanted_params = len(frozen_param_shapes)
227
- wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
- avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
- print(f'Frozen params: Have {avail_numel} numels to process.')
230
- print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
-
232
- total_params = 0
233
- total_numel = 0
234
- for name, shape in frozen_param_shapes.items():
235
- total_params += 1
236
- unpartitioned_numel = shape.numel()
237
- total_numel += unpartitioned_numel
238
-
239
- state_dict[name] = frozen_param_fragments[name]
240
-
241
- if debug:
242
- print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
-
244
- print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
-
246
-
247
- def _has_callable(obj, fn):
248
- attr = getattr(obj, fn, None)
249
- return callable(attr)
250
-
251
-
252
- def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
- param_shapes = zero_model_states[0].param_shapes
254
-
255
- # Reconstruction protocol:
256
- #
257
- # XXX: document this
258
-
259
- if debug:
260
- for i in range(world_size):
261
- for j in range(len(fp32_flat_groups[0])):
262
- print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
-
264
- # XXX: memory usage doubles here (zero2)
265
- num_param_groups = len(fp32_flat_groups[0])
266
- merged_single_partition_of_fp32_groups = []
267
- for i in range(num_param_groups):
268
- merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
- full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
- merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
- avail_numel = sum(
272
- [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
-
274
- if debug:
275
- wanted_params = sum([len(shapes) for shapes in param_shapes])
276
- wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
- # not asserting if there is a mismatch due to possible padding
278
- print(f"Have {avail_numel} numels to process.")
279
- print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
-
281
- # params
282
- # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
- # out-of-core computing solution
284
- total_numel = 0
285
- total_params = 0
286
- for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
- offset = 0
288
- avail_numel = full_single_fp32_vector.numel()
289
- for name, shape in shapes.items():
290
-
291
- unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
- total_numel += unpartitioned_numel
293
- total_params += 1
294
-
295
- if debug:
296
- print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
- state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
- offset += unpartitioned_numel
299
-
300
- # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
- # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
- # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
- # live optimizer object, so we are checking that the numbers are within the right range
304
- align_to = 2 * world_size
305
-
306
- def zero2_align(x):
307
- return align_to * math.ceil(x / align_to)
308
-
309
- if debug:
310
- print(f"original offset={offset}, avail_numel={avail_numel}")
311
-
312
- offset = zero2_align(offset)
313
- avail_numel = zero2_align(avail_numel)
314
-
315
- if debug:
316
- print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
-
318
- # Sanity check
319
- if offset != avail_numel:
320
- raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
-
322
- print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
-
324
-
325
- def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
- exclude_frozen_parameters):
327
- state_dict = OrderedDict()
328
-
329
- # buffers
330
- buffers = zero_model_states[0].buffers
331
- state_dict.update(buffers)
332
- if debug:
333
- print(f"added {len(buffers)} buffers")
334
-
335
- if not exclude_frozen_parameters:
336
- _zero2_merge_frozen_params(state_dict, zero_model_states)
337
-
338
- _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
-
340
- # recover shared parameters
341
- for pair in zero_model_states[0].shared_params:
342
- if pair[1] in state_dict:
343
- state_dict[pair[0]] = state_dict[pair[1]]
344
-
345
- return state_dict
346
-
347
-
348
- def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
- remainder = unpartitioned_numel % world_size
350
- padding_numel = (world_size - remainder) if remainder else 0
351
- partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
- return partitioned_numel, padding_numel
353
-
354
-
355
- def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
- if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
- return
358
-
359
- if debug:
360
- for i in range(world_size):
361
- num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
- print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
-
364
- frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
- wanted_params = len(frozen_param_shapes)
366
- wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
- avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
- print(f'Frozen params: Have {avail_numel} numels to process.')
369
- print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
-
371
- total_params = 0
372
- total_numel = 0
373
- for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
- total_params += 1
375
- unpartitioned_numel = shape.numel()
376
- total_numel += unpartitioned_numel
377
-
378
- param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
- state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
-
381
- partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
-
383
- if debug:
384
- print(
385
- f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
- )
387
-
388
- print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
-
390
-
391
- class GatheredTensor:
392
- """
393
- A pseudo tensor that collects partitioned weights.
394
- It is more memory efficient when there are multiple groups.
395
- """
396
-
397
- def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
- self.flat_groups = flat_groups
399
- self.flat_groups_offset = flat_groups_offset
400
- self.offset = offset
401
- self.partitioned_numel = partitioned_numel
402
- self.shape = shape
403
- self.dtype = self.flat_groups[0][0].dtype
404
-
405
- def contiguous(self):
406
- """
407
- Merge partitioned weights from flat_groups into a single tensor.
408
- """
409
- end_idx = self.offset + self.partitioned_numel
410
- world_size = len(self.flat_groups)
411
- pad_flat_param_chunks = []
412
-
413
- for rank_i in range(world_size):
414
- # for each rank, we need to collect weights from related group/groups
415
- flat_groups_at_rank_i = self.flat_groups[rank_i]
416
- start_group_id = None
417
- end_group_id = None
418
- for group_id in range(len(self.flat_groups_offset)):
419
- if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
- start_group_id = group_id
421
- if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
- end_group_id = group_id
423
- break
424
- # collect weights from related group/groups
425
- for group_id in range(start_group_id, end_group_id + 1):
426
- flat_tensor = flat_groups_at_rank_i[group_id]
427
- start_offset = self.offset - self.flat_groups_offset[group_id]
428
- end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
- pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
-
431
- # collect weights from all ranks
432
- pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
- param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
- return param
435
-
436
-
437
- def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
- param_shapes = zero_model_states[0].param_shapes
439
- avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
-
441
- # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
- # param, re-consolidating each param, while dealing with padding if any
443
-
444
- # merge list of dicts, preserving order
445
- param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
-
447
- if debug:
448
- for i in range(world_size):
449
- print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
-
451
- wanted_params = len(param_shapes)
452
- wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
- # not asserting if there is a mismatch due to possible padding
454
- avail_numel = fp32_flat_groups[0].numel() * world_size
455
- print(f"Trainable params: Have {avail_numel} numels to process.")
456
- print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
-
458
- # params
459
- # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
- # out-of-core computing solution
461
- offset = 0
462
- total_numel = 0
463
- total_params = 0
464
- flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
- for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
- unpartitioned_numel = shape.numel()
467
- total_numel += unpartitioned_numel
468
- total_params += 1
469
- partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
-
471
- if debug:
472
- print(
473
- f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
- )
475
-
476
- # memory efficient tensor
477
- tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
- state_dict[name] = tensor
479
- offset += partitioned_numel
480
-
481
- offset *= world_size
482
-
483
- # Sanity check
484
- if offset != avail_numel:
485
- raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
-
487
- print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
-
489
-
490
- def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
- exclude_frozen_parameters):
492
- state_dict = OrderedDict()
493
-
494
- # buffers
495
- buffers = zero_model_states[0].buffers
496
- state_dict.update(buffers)
497
- if debug:
498
- print(f"added {len(buffers)} buffers")
499
-
500
- if not exclude_frozen_parameters:
501
- _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
-
503
- _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
-
505
- # recover shared parameters
506
- for pair in zero_model_states[0].shared_params:
507
- if pair[1] in state_dict:
508
- state_dict[pair[0]] = state_dict[pair[1]]
509
-
510
- return state_dict
511
-
512
-
513
- def to_torch_tensor(state_dict, return_empty_tensor=False):
514
- """
515
- Convert state_dict of GatheredTensor to torch tensor
516
- """
517
- torch_state_dict = {}
518
- converted_tensors = {}
519
- for name, tensor in state_dict.items():
520
- tensor_id = id(tensor)
521
- if tensor_id in converted_tensors: # shared tensors
522
- shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
- torch_state_dict[name] = shared_tensor
524
- else:
525
- converted_tensors[tensor_id] = name
526
- if return_empty_tensor:
527
- torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
- else:
529
- torch_state_dict[name] = tensor.contiguous()
530
- return torch_state_dict
531
-
532
-
533
- def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
- tag=None,
535
- exclude_frozen_parameters=False,
536
- lazy_mode=False):
537
- """
538
- Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
- ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
- via a model hub.
541
-
542
- Args:
543
- - ``checkpoint_dir``: path to the desired checkpoint folder
544
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
- - ``exclude_frozen_parameters``: exclude frozen parameters
546
- - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
- Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
-
549
- Returns:
550
- - pytorch ``state_dict``
551
-
552
- A typical usage might be ::
553
-
554
- from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
- # do the training and checkpoint saving
556
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
- model = model.cpu() # move to cpu
558
- model.load_state_dict(state_dict)
559
- # submit to model hub or save the model to share with others
560
-
561
- In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
- application. i.e. you will need to re-initialize the deepspeed engine, since
563
- ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
-
565
- If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
-
567
- Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
- You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
- the checkpoint. Or you can load state_dict in lazy mode ::
570
-
571
- from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
- for name, lazy_tensor in state_dict.item():
574
- tensor = lazy_tensor.contiguous() # to cpu
575
- print(name, tensor)
576
- # del tensor to release memory if it no longer in use
577
- """
578
- if tag is None:
579
- latest_path = os.path.join(checkpoint_dir, 'latest')
580
- if os.path.isfile(latest_path):
581
- with open(latest_path, 'r') as fd:
582
- tag = fd.read().strip()
583
- else:
584
- raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
-
586
- ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
-
588
- if not os.path.isdir(ds_checkpoint_dir):
589
- raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
-
591
- state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
- if lazy_mode:
593
- return state_dict
594
- else:
595
- return to_torch_tensor(state_dict)
596
-
597
-
598
- def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
- output_dir,
600
- max_shard_size="5GB",
601
- safe_serialization=False,
602
- tag=None,
603
- exclude_frozen_parameters=False):
604
- """
605
- Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
- loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
-
608
- Args:
609
- - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
- - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
- - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
- - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
- - ``exclude_frozen_parameters``: exclude frozen parameters
615
- """
616
-
617
- # Dependency pre-check
618
- if safe_serialization:
619
- try:
620
- from safetensors.torch import save_file
621
- except ImportError:
622
- print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
- raise
624
- if max_shard_size is not None:
625
- try:
626
- from huggingface_hub import split_torch_state_dict_into_shards
627
- except ImportError:
628
- print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
- raise
630
-
631
- # Convert zero checkpoint to state_dict
632
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
- tag,
634
- exclude_frozen_parameters,
635
- lazy_mode=True)
636
-
637
- # Shard the model if it is too big.
638
- weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
- if max_shard_size is not None:
640
- filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
- # an memory-efficient approach for sharding
642
- empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
- state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
- filename_pattern=filename_pattern,
645
- max_shard_size=max_shard_size)
646
- else:
647
- from collections import namedtuple
648
- StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
- state_dict_split = StateDictSplit(is_sharded=False,
650
- filename_to_tensors={weights_name: list(state_dict.keys())})
651
-
652
- # Save the model by shard
653
- os.makedirs(output_dir, exist_ok=True)
654
- filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
- for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
- shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
- shard_state_dict = to_torch_tensor(shard_state_dict)
658
- output_path = os.path.join(output_dir, shard_file)
659
- if safe_serialization:
660
- save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
- else:
662
- torch.save(shard_state_dict, output_path)
663
- # release the memory of current shard
664
- for tensor_name in list(shard_state_dict.keys()):
665
- del state_dict[tensor_name]
666
- del shard_state_dict[tensor_name]
667
- del shard_state_dict
668
- gc.collect()
669
-
670
- # Save index if sharded
671
- if state_dict_split.is_sharded:
672
- index = {
673
- "metadata": state_dict_split.metadata,
674
- "weight_map": state_dict_split.tensor_to_filename,
675
- }
676
- save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
- save_index_file = os.path.join(output_dir, save_index_file)
678
- with open(save_index_file, "w", encoding="utf-8") as f:
679
- content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
- f.write(content)
681
-
682
-
683
- def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
- """
685
- 1. Put the provided model to cpu
686
- 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
- 3. Load it into the provided model
688
-
689
- Args:
690
- - ``model``: the model object to update
691
- - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
-
694
- Returns:
695
- - ``model`: modified model
696
-
697
- Make sure you have plenty of CPU memory available before you call this function. If you don't
698
- have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
- conveniently placed for you in the checkpoint folder.
700
-
701
- A typical usage might be ::
702
-
703
- from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
- model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
- # submit to model hub or save the model to share with others
706
-
707
- Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
- of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
- ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
-
711
- """
712
- logger.info(f"Extracting fp32 weights")
713
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
-
715
- logger.info(f"Overwriting model with fp32 weights")
716
- model = model.cpu()
717
- model.load_state_dict(state_dict, strict=False)
718
-
719
- return model
720
-
721
-
722
- if __name__ == "__main__":
723
- parser = argparse.ArgumentParser()
724
- parser.add_argument("checkpoint_dir",
725
- type=str,
726
- help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
- parser.add_argument("output_dir",
728
- type=str,
729
- help="directory to the pytorch fp32 state_dict output files"
730
- "(e.g. path/checkpoint-12-output/)")
731
- parser.add_argument(
732
- "--max_shard_size",
733
- type=str,
734
- default="5GB",
735
- help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
- "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
- "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
- "without CPU OOM issues.")
739
- parser.add_argument(
740
- "--safe_serialization",
741
- default=False,
742
- action='store_true',
743
- help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
- parser.add_argument("-t",
745
- "--tag",
746
- type=str,
747
- default=None,
748
- help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
- parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
- parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
- args = parser.parse_args()
752
-
753
- debug = args.debug
754
-
755
- convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
- args.output_dir,
757
- max_shard_size=args.max_shard_size,
758
- safe_serialization=args.safe_serialization,
759
- tag=args.tag,
760
- exclude_frozen_parameters=args.exclude_frozen_parameters)