Hugging Face's logo

RuoliuYang
/
analysis

Safetensors
Model card Files
xet
 Community
RuoliuYang commited on
Commit
2ecc10b
·
verified ·
1 Parent(s): 34807ae

cleanup stage2: drop intermediate checkpoint-1200 (keep best 1250)

Browse files
Files changed (38) hide show
  1. size-80k/checkpoint-1200/added_tokens.json +0 -29
  2. size-80k/checkpoint-1200/chat_template.jinja +0 -7
  3. size-80k/checkpoint-1200/config.json +0 -144
  4. size-80k/checkpoint-1200/generation_config.json +0 -12
  5. size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +0 -3
  6. size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +0 -3
  7. size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt +0 -3
  8. size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt +0 -3
  9. size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt +0 -3
  10. size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt +0 -3
  11. size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt +0 -3
  12. size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt +0 -3
  13. size-80k/checkpoint-1200/global_step1200/mp_rank_00_model_states.pt +0 -3
  14. size-80k/checkpoint-1200/latest +0 -1
  15. size-80k/checkpoint-1200/merges.txt +0 -0
  16. size-80k/checkpoint-1200/model-00001-of-00004.safetensors +0 -3
  17. size-80k/checkpoint-1200/model-00002-of-00004.safetensors +0 -3
  18. size-80k/checkpoint-1200/model-00003-of-00004.safetensors +0 -3
  19. size-80k/checkpoint-1200/model-00004-of-00004.safetensors +0 -3
  20. size-80k/checkpoint-1200/model.safetensors.index.json +0 -737
  21. size-80k/checkpoint-1200/preprocessor_config.json +0 -37
  22. size-80k/checkpoint-1200/rng_state_0.pth +0 -3
  23. size-80k/checkpoint-1200/rng_state_1.pth +0 -3
  24. size-80k/checkpoint-1200/rng_state_2.pth +0 -3
  25. size-80k/checkpoint-1200/rng_state_3.pth +0 -3
  26. size-80k/checkpoint-1200/rng_state_4.pth +0 -3
  27. size-80k/checkpoint-1200/rng_state_5.pth +0 -3
  28. size-80k/checkpoint-1200/rng_state_6.pth +0 -3
  29. size-80k/checkpoint-1200/rng_state_7.pth +0 -3
  30. size-80k/checkpoint-1200/scheduler.pt +0 -3
  31. size-80k/checkpoint-1200/special_tokens_map.json +0 -31
  32. size-80k/checkpoint-1200/tokenizer.json +0 -3
  33. size-80k/checkpoint-1200/tokenizer_config.json +0 -249
  34. size-80k/checkpoint-1200/trainer_state.json +0 -1354
  35. size-80k/checkpoint-1200/training_args.bin +0 -3
  36. size-80k/checkpoint-1200/video_preprocessor_config.json +0 -43
  37. size-80k/checkpoint-1200/vocab.json +0 -0
  38. size-80k/checkpoint-1200/zero_to_fp32.py +0 -760
size-80k/checkpoint-1200/added_tokens.json DELETED
@@ -1,29 +0,0 @@
1
- {
2
- "</abs_vis_token>": 151667,
3
- "</observation>": 151669,
4
- "</tool_call>": 151658,
5
- "<abs_vis_token>": 151666,
6
- "<abs_vis_token_pad>": 151665,
7
- "<observation>": 151668,
8
- "<tool_call>": 151657,
9
- "<|box_end|>": 151649,
10
- "<|box_start|>": 151648,
11
- "<|endoftext|>": 151643,
12
- "<|file_sep|>": 151664,
13
- "<|fim_middle|>": 151660,
14
- "<|fim_pad|>": 151662,
15
- "<|fim_prefix|>": 151659,
16
- "<|fim_suffix|>": 151661,
17
- "<|im_end|>": 151645,
18
- "<|im_start|>": 151644,
19
- "<|image_pad|>": 151655,
20
- "<|object_ref_end|>": 151647,
21
- "<|object_ref_start|>": 151646,
22
- "<|quad_end|>": 151651,
23
- "<|quad_start|>": 151650,
24
- "<|repo_name|>": 151663,
25
- "<|video_pad|>": 151656,
26
- "<|vision_end|>": 151653,
27
- "<|vision_pad|>": 151654,
28
- "<|vision_start|>": 151652
29
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/chat_template.jinja DELETED
@@ -1,7 +0,0 @@
1
- {% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
2
- You are a helpful assistant.<|im_end|>
3
- {% endif %}<|im_start|>{{ message['role'] }}
4
- {% if message['content'] is string %}{{ message['content'] }}<|im_end|>
5
- {% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
6
- {% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
7
- {% endif %}
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/config.json DELETED
@@ -1,144 +0,0 @@
1
- {
2
- "answer_start_pattern": [
3
- 151644,
4
- 77091
5
- ],
6
- "architectures": [
7
- "Qwen2_5_VLForConditionalGeneration"
8
- ],
9
- "attention_dropout": 0.0,
10
- "bos_token_id": 151643,
11
- "eos_token_id": 151645,
12
- "hidden_act": "silu",
13
- "hidden_size": 3584,
14
- "image_token_id": 151655,
15
- "initializer_range": 0.02,
16
- "intermediate_size": 18944,
17
- "latent_end_id": 151667,
18
- "latent_start_id": 151666,
19
- "latent_token_id": 151665,
20
- "loss_type": "ForCausalLMLoss",
21
- "max_position_embeddings": 128000,
22
- "max_window_layers": 28,
23
- "model_type": "qwen2_5_vl",
24
- "num_attention_heads": 28,
25
- "num_hidden_layers": 28,
26
- "num_key_value_heads": 4,
27
- "rms_norm_eps": 1e-06,
28
- "rope_scaling": {
29
- "mrope_section": [
30
- 16,
31
- 24,
32
- 24
33
- ],
34
- "rope_type": "default",
35
- "type": "default"
36
- },
37
- "rope_theta": 1000000.0,
38
- "sliding_window": 32768,
39
- "stage": "sft_stage2",
40
- "text_config": {
41
- "architectures": [
42
- "Qwen2_5_VLForConditionalGeneration"
43
- ],
44
- "attention_dropout": 0.0,
45
- "bos_token_id": 151643,
46
- "eos_token_id": 151645,
47
- "hidden_act": "silu",
48
- "hidden_size": 3584,
49
- "image_token_id": null,
50
- "initializer_range": 0.02,
51
- "intermediate_size": 18944,
52
- "layer_types": [
53
- "full_attention",
54
- "full_attention",
55
- "full_attention",
56
- "full_attention",
57
- "full_attention",
58
- "full_attention",
59
- "full_attention",
60
- "full_attention",
61
- "full_attention",
62
- "full_attention",
63
- "full_attention",
64
- "full_attention",
65
- "full_attention",
66
- "full_attention",
67
- "full_attention",
68
- "full_attention",
69
- "full_attention",
70
- "full_attention",
71
- "full_attention",
72
- "full_attention",
73
- "full_attention",
74
- "full_attention",
75
- "full_attention",
76
- "full_attention",
77
- "full_attention",
78
- "full_attention",
79
- "full_attention",
80
- "full_attention"
81
- ],
82
- "max_position_embeddings": 128000,
83
- "max_window_layers": 28,
84
- "model_type": "qwen2_5_vl_text",
85
- "num_attention_heads": 28,
86
- "num_hidden_layers": 28,
87
- "num_key_value_heads": 4,
88
- "rms_norm_eps": 1e-06,
89
- "rope_scaling": {
90
- "mrope_section": [
91
- 16,
92
- 24,
93
- 24
94
- ],
95
- "rope_type": "default",
96
- "type": "default"
97
- },
98
- "rope_theta": 1000000.0,
99
- "sliding_window": null,
100
- "torch_dtype": "bfloat16",
101
- "use_cache": true,
102
- "use_sliding_window": false,
103
- "video_token_id": null,
104
- "vision_end_token_id": 151653,
105
- "vision_start_token_id": 151652,
106
- "vision_token_id": 151654,
107
- "vocab_size": 151670
108
- },
109
- "tie_word_embeddings": false,
110
- "torch_dtype": "bfloat16",
111
- "transformers_version": "4.54.0",
112
- "use_cache": false,
113
- "use_sliding_window": false,
114
- "video_token_id": 151656,
115
- "vision_config": {
116
- "depth": 32,
117
- "fullatt_block_indexes": [
118
- 7,
119
- 15,
120
- 23,
121
- 31
122
- ],
123
- "hidden_act": "silu",
124
- "hidden_size": 1280,
125
- "in_channels": 3,
126
- "in_chans": 3,
127
- "initializer_range": 0.02,
128
- "intermediate_size": 3420,
129
- "model_type": "qwen2_5_vl",
130
- "num_heads": 16,
131
- "out_hidden_size": 3584,
132
- "patch_size": 14,
133
- "spatial_merge_size": 2,
134
- "spatial_patch_size": 14,
135
- "temporal_patch_size": 2,
136
- "tokens_per_second": 2,
137
- "torch_dtype": "bfloat16",
138
- "window_size": 112
139
- },
140
- "vision_end_token_id": 151653,
141
- "vision_start_token_id": 151652,
142
- "vision_token_id": 151654,
143
- "vocab_size": 151670
144
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/generation_config.json DELETED
@@ -1,12 +0,0 @@
1
- {
2
- "bos_token_id": 151643,
3
- "do_sample": true,
4
- "eos_token_id": [
5
- 151645,
6
- 151643
7
- ],
8
- "pad_token_id": 151643,
9
- "repetition_penalty": 1.05,
10
- "temperature": 1e-06,
11
- "transformers_version": "4.54.0"
12
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7bf8b77e272dff0a1b6ab745ce65f027e0985b7eaa9bdb719e543423d53b6034
3
- size 11419197025
 
 
 
 
size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a64443a83f05da2fc7f40583843275fea46655696298e4ce0802c9c8938de12c
3
- size 11419198241
 
 
 
 
size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c2662a68bb9537179967a75d88463de0dfc38761ee6d7bdd7d5c3fdf87339007
3
- size 11419198305
 
 
 
 
size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6b0a7f3744d295d5b32c6aec8c213d8e2316cc26f08e9eb48de230f37bcd528b
3
- size 11419198305
 
 
 
 
size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:90324f13efcbc29ec8b21b0744c9dcb003661db5b40e8920994b39ea9eab4080
3
- size 11419198689
 
 
 
 
size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f2bf2a0ad7519913b75f8704cb79fe97f474b5142a600dbb914bac159e0166c1
3
- size 11419198369
 
 
 
 
size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:49bc1195585963f494912db1116590afdfc6865cb8d4c728f5a1ff4bc45118fa
3
- size 11419198497
 
 
 
 
size-80k/checkpoint-1200/global_step1200/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:499ef7542a082f30a6d0f851c5c80a6e90f0f40ae9c3960d9af8736b213000da
3
- size 11419196449
 
 
 
 
size-80k/checkpoint-1200/global_step1200/mp_rank_00_model_states.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9239ad6f2a319f0e5aefc559b992ef224ea4d99dd002071380254cbc3913a438
3
- size 17932200235
 
 
 
 
size-80k/checkpoint-1200/latest DELETED
@@ -1 +0,0 @@
1
- global_step1200
 
 
size-80k/checkpoint-1200/merges.txt DELETED
The diff for this file is too large to render. See raw diff
 
size-80k/checkpoint-1200/model-00001-of-00004.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:71880829ed19aaded33f16c0efa0be0dfda83866bf0bb0d81bd170f7af658fbf
3
- size 4965419112
 
 
 
 
size-80k/checkpoint-1200/model-00002-of-00004.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9abcc7c31b4c5c757bc1ca440c42760d8a7f51ff936bcd5e6b2c0fa570140725
3
- size 4991495816
 
 
 
 
size-80k/checkpoint-1200/model-00003-of-00004.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d3f58133ec398f9054244e7a618b22c4da638a7b6a48db8fb516c485af58cbb1
3
- size 4932751040
 
 
 
 
size-80k/checkpoint-1200/model-00004-of-00004.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5795aa74a08bc719374cf66e2c243be00f1d286c1af7539f06fa94e36f3180b0
3
- size 1689100192
 
 
 
 
size-80k/checkpoint-1200/model.safetensors.index.json DELETED
@@ -1,737 +0,0 @@
1
- {
2
- "metadata": {
3
- "total_parameters": 8289342464,
4
- "total_size": 16578684928
5
- },
6
- "weight_map": {
7
- "lm_head.weight": "model-00004-of-00004.safetensors",
8
- "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
9
- "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
10
- "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
11
- "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
12
- "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
13
- "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
14
- "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
15
- "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
16
- "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
17
- "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
18
- "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
19
- "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
20
- "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
21
- "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
22
- "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
23
- "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
24
- "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
25
- "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
26
- "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
27
- "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
28
- "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
29
- "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
30
- "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
31
- "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
32
- "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
33
- "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
34
- "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
35
- "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
36
- "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
37
- "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
38
- "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
39
- "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
40
- "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
41
- "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
42
- "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
43
- "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
44
- "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
45
- "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
46
- "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
47
- "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
48
- "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
49
- "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
50
- "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
51
- "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
52
- "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
53
- "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
54
- "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
55
- "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
56
- "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
57
- "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
58
- "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
59
- "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
60
- "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
61
- "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
62
- "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
63
- "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
64
- "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
65
- "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
66
- "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
67
- "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
68
- "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
69
- "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
70
- "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
71
- "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
72
- "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
73
- "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
74
- "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
75
- "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
76
- "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
77
- "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
78
- "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
79
- "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
80
- "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
81
- "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
82
- "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
83
- "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
84
- "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
85
- "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
86
- "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
87
- "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
88
- "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
89
- "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
90
- "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
91
- "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
92
- "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
93
- "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
94
- "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
95
- "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
96
- "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
97
- "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
98
- "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
99
- "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
100
- "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
101
- "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
102
- "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
103
- "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
104
- "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
105
- "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
106
- "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
107
- "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
108
- "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
109
- "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
110
- "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
111
- "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
112
- "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
113
- "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
114
- "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
115
- "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
116
- "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
117
- "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
118
- "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
119
- "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
120
- "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
121
- "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
122
- "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
123
- "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
124
- "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
125
- "model.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
126
- "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
127
- "model.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
128
- "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
129
- "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
130
- "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
131
- "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
132
- "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
133
- "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
134
- "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
135
- "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
136
- "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
137
- "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
138
- "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
139
- "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
140
- "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
141
- "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
142
- "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
143
- "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
144
- "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
145
- "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
146
- "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
147
- "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
148
- "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
149
- "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
150
- "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
151
- "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
152
- "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
153
- "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
154
- "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
155
- "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
156
- "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
157
- "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
158
- "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
159
- "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
160
- "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
161
- "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
162
- "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
163
- "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
164
- "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
165
- "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
166
- "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
167
- "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
168
- "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
169
- "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
170
- "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
171
- "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
172
- "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
173
- "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
174
- "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
175
- "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
176
- "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
177
- "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
178
- "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
179
- "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
180
- "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
181
- "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
182
- "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
183
- "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
184
- "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
185
- "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
186
- "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
187
- "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
188
- "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
189
- "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
190
- "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
191
- "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
192
- "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
193
- "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
194
- "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
195
- "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
196
- "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
197
- "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
198
- "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
199
- "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
200
- "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
201
- "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
202
- "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
203
- "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
204
- "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
205
- "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
206
- "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
207
- "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
208
- "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
209
- "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
210
- "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
211
- "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
212
- "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
213
- "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
214
- "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
215
- "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
216
- "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
217
- "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
218
- "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
219
- "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
220
- "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
221
- "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
222
- "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
223
- "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
224
- "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
225
- "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
226
- "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
227
- "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
228
- "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
229
- "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
230
- "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
231
- "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
232
- "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
233
- "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
234
- "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
235
- "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
236
- "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
237
- "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
238
- "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
239
- "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
240
- "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
241
- "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
242
- "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
243
- "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
244
- "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
245
- "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
246
- "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
247
- "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
248
- "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
249
- "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
250
- "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
251
- "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
252
- "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
253
- "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
254
- "model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
255
- "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
256
- "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
257
- "model.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
258
- "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
259
- "model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
260
- "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
261
- "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
262
- "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
263
- "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
264
- "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
265
- "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
266
- "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
267
- "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
268
- "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
269
- "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
270
- "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
271
- "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
272
- "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
273
- "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
274
- "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
275
- "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
276
- "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
277
- "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
278
- "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
279
- "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
280
- "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
281
- "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
282
- "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
283
- "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
284
- "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
285
- "model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
286
- "model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
287
- "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
288
- "model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
289
- "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
290
- "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
291
- "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
292
- "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
293
- "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
294
- "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
295
- "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
296
- "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
297
- "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
298
- "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
299
- "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
300
- "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
301
- "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
302
- "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
303
- "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
304
- "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
305
- "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
306
- "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
307
- "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
308
- "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
309
- "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
310
- "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
311
- "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
312
- "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
313
- "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
314
- "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
315
- "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
316
- "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
317
- "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
318
- "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
319
- "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
320
- "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
321
- "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
322
- "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
323
- "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
324
- "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
325
- "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
326
- "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
327
- "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
328
- "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
329
- "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
330
- "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
331
- "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
332
- "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
333
- "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
334
- "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
335
- "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
336
- "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
337
- "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
338
- "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
339
- "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
340
- "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
341
- "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
342
- "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
343
- "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
344
- "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
345
- "model.norm.weight": "model-00004-of-00004.safetensors",
346
- "visual.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
347
- "visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
348
- "visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
349
- "visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
350
- "visual.blocks.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
351
- "visual.blocks.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
352
- "visual.blocks.0.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
353
- "visual.blocks.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
354
- "visual.blocks.0.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
355
- "visual.blocks.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
356
- "visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
357
- "visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
358
- "visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
359
- "visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
360
- "visual.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
361
- "visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
362
- "visual.blocks.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
363
- "visual.blocks.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
364
- "visual.blocks.1.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
365
- "visual.blocks.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
366
- "visual.blocks.1.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
367
- "visual.blocks.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
368
- "visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
369
- "visual.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
370
- "visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
371
- "visual.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
372
- "visual.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
373
- "visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
374
- "visual.blocks.10.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
375
- "visual.blocks.10.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
376
- "visual.blocks.10.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
377
- "visual.blocks.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
378
- "visual.blocks.10.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
379
- "visual.blocks.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
380
- "visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
381
- "visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
382
- "visual.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
383
- "visual.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
384
- "visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
385
- "visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
386
- "visual.blocks.11.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
387
- "visual.blocks.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
388
- "visual.blocks.11.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
389
- "visual.blocks.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
390
- "visual.blocks.11.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
391
- "visual.blocks.11.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
392
- "visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
393
- "visual.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
394
- "visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
395
- "visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
396
- "visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
397
- "visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
398
- "visual.blocks.12.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
399
- "visual.blocks.12.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
400
- "visual.blocks.12.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
401
- "visual.blocks.12.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
402
- "visual.blocks.12.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
403
- "visual.blocks.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
404
- "visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
405
- "visual.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
406
- "visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
407
- "visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
408
- "visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
409
- "visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
410
- "visual.blocks.13.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
411
- "visual.blocks.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
412
- "visual.blocks.13.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
413
- "visual.blocks.13.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
414
- "visual.blocks.13.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
415
- "visual.blocks.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
416
- "visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
417
- "visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
418
- "visual.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
419
- "visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
420
- "visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
421
- "visual.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
422
- "visual.blocks.14.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
423
- "visual.blocks.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
424
- "visual.blocks.14.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
425
- "visual.blocks.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
426
- "visual.blocks.14.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
427
- "visual.blocks.14.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
428
- "visual.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
429
- "visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
430
- "visual.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
431
- "visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
432
- "visual.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
433
- "visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
434
- "visual.blocks.15.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
435
- "visual.blocks.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
436
- "visual.blocks.15.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
437
- "visual.blocks.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
438
- "visual.blocks.15.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
439
- "visual.blocks.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
440
- "visual.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
441
- "visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
442
- "visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
443
- "visual.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
444
- "visual.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
445
- "visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
446
- "visual.blocks.16.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
447
- "visual.blocks.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
448
- "visual.blocks.16.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
449
- "visual.blocks.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
450
- "visual.blocks.16.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
451
- "visual.blocks.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
452
- "visual.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
453
- "visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
454
- "visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
455
- "visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
456
- "visual.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
457
- "visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
458
- "visual.blocks.17.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
459
- "visual.blocks.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
460
- "visual.blocks.17.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
461
- "visual.blocks.17.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
462
- "visual.blocks.17.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
463
- "visual.blocks.17.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
464
- "visual.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
465
- "visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
466
- "visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
467
- "visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
468
- "visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
469
- "visual.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
470
- "visual.blocks.18.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
471
- "visual.blocks.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
472
- "visual.blocks.18.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
473
- "visual.blocks.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
474
- "visual.blocks.18.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
475
- "visual.blocks.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
476
- "visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
477
- "visual.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
478
- "visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
479
- "visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
480
- "visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
481
- "visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
482
- "visual.blocks.19.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
483
- "visual.blocks.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
484
- "visual.blocks.19.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
485
- "visual.blocks.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
486
- "visual.blocks.19.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
487
- "visual.blocks.19.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
488
- "visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
489
- "visual.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
490
- "visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
491
- "visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
492
- "visual.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
493
- "visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
494
- "visual.blocks.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
495
- "visual.blocks.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
496
- "visual.blocks.2.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
497
- "visual.blocks.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
498
- "visual.blocks.2.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
499
- "visual.blocks.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
500
- "visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
501
- "visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
502
- "visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
503
- "visual.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
504
- "visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
505
- "visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
506
- "visual.blocks.20.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
507
- "visual.blocks.20.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
508
- "visual.blocks.20.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
509
- "visual.blocks.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
510
- "visual.blocks.20.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
511
- "visual.blocks.20.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
512
- "visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
513
- "visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
514
- "visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
515
- "visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
516
- "visual.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
517
- "visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
518
- "visual.blocks.21.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
519
- "visual.blocks.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
520
- "visual.blocks.21.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
521
- "visual.blocks.21.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
522
- "visual.blocks.21.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
523
- "visual.blocks.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
524
- "visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
525
- "visual.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
526
- "visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
527
- "visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
528
- "visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
529
- "visual.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
530
- "visual.blocks.22.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
531
- "visual.blocks.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
532
- "visual.blocks.22.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
533
- "visual.blocks.22.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
534
- "visual.blocks.22.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
535
- "visual.blocks.22.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
536
- "visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
537
- "visual.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
538
- "visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
539
- "visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
540
- "visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
541
- "visual.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
542
- "visual.blocks.23.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
543
- "visual.blocks.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
544
- "visual.blocks.23.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
545
- "visual.blocks.23.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
546
- "visual.blocks.23.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
547
- "visual.blocks.23.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
548
- "visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
549
- "visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
550
- "visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
551
- "visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
552
- "visual.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
553
- "visual.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
554
- "visual.blocks.24.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
555
- "visual.blocks.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
556
- "visual.blocks.24.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
557
- "visual.blocks.24.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
558
- "visual.blocks.24.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
559
- "visual.blocks.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
560
- "visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
561
- "visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
562
- "visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
563
- "visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
564
- "visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
565
- "visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
566
- "visual.blocks.25.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
567
- "visual.blocks.25.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
568
- "visual.blocks.25.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
569
- "visual.blocks.25.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
570
- "visual.blocks.25.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
571
- "visual.blocks.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
572
- "visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
573
- "visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
574
- "visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
575
- "visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
576
- "visual.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
577
- "visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
578
- "visual.blocks.26.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
579
- "visual.blocks.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
580
- "visual.blocks.26.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
581
- "visual.blocks.26.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
582
- "visual.blocks.26.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
583
- "visual.blocks.26.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
584
- "visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
585
- "visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
586
- "visual.blocks.27.attn.proj.bias": "model-00001-of-00004.safetensors",
587
- "visual.blocks.27.attn.proj.weight": "model-00001-of-00004.safetensors",
588
- "visual.blocks.27.attn.qkv.bias": "model-00001-of-00004.safetensors",
589
- "visual.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors",
590
- "visual.blocks.27.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
591
- "visual.blocks.27.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
592
- "visual.blocks.27.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
593
- "visual.blocks.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
594
- "visual.blocks.27.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
595
- "visual.blocks.27.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
596
- "visual.blocks.27.norm1.weight": "model-00001-of-00004.safetensors",
597
- "visual.blocks.27.norm2.weight": "model-00001-of-00004.safetensors",
598
- "visual.blocks.28.attn.proj.bias": "model-00001-of-00004.safetensors",
599
- "visual.blocks.28.attn.proj.weight": "model-00001-of-00004.safetensors",
600
- "visual.blocks.28.attn.qkv.bias": "model-00001-of-00004.safetensors",
601
- "visual.blocks.28.attn.qkv.weight": "model-00001-of-00004.safetensors",
602
- "visual.blocks.28.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
603
- "visual.blocks.28.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
604
- "visual.blocks.28.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
605
- "visual.blocks.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
606
- "visual.blocks.28.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
607
- "visual.blocks.28.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
608
- "visual.blocks.28.norm1.weight": "model-00001-of-00004.safetensors",
609
- "visual.blocks.28.norm2.weight": "model-00001-of-00004.safetensors",
610
- "visual.blocks.29.attn.proj.bias": "model-00001-of-00004.safetensors",
611
- "visual.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors",
612
- "visual.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors",
613
- "visual.blocks.29.attn.qkv.weight": "model-00001-of-00004.safetensors",
614
- "visual.blocks.29.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
615
- "visual.blocks.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
616
- "visual.blocks.29.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
617
- "visual.blocks.29.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
618
- "visual.blocks.29.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
619
- "visual.blocks.29.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
620
- "visual.blocks.29.norm1.weight": "model-00001-of-00004.safetensors",
621
- "visual.blocks.29.norm2.weight": "model-00001-of-00004.safetensors",
622
- "visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
623
- "visual.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
624
- "visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
625
- "visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
626
- "visual.blocks.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
627
- "visual.blocks.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
628
- "visual.blocks.3.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
629
- "visual.blocks.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
630
- "visual.blocks.3.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
631
- "visual.blocks.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
632
- "visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
633
- "visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
634
- "visual.blocks.30.attn.proj.bias": "model-00001-of-00004.safetensors",
635
- "visual.blocks.30.attn.proj.weight": "model-00001-of-00004.safetensors",
636
- "visual.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors",
637
- "visual.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors",
638
- "visual.blocks.30.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
639
- "visual.blocks.30.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
640
- "visual.blocks.30.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
641
- "visual.blocks.30.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
642
- "visual.blocks.30.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
643
- "visual.blocks.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
644
- "visual.blocks.30.norm1.weight": "model-00001-of-00004.safetensors",
645
- "visual.blocks.30.norm2.weight": "model-00001-of-00004.safetensors",
646
- "visual.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors",
647
- "visual.blocks.31.attn.proj.weight": "model-00001-of-00004.safetensors",
648
- "visual.blocks.31.attn.qkv.bias": "model-00001-of-00004.safetensors",
649
- "visual.blocks.31.attn.qkv.weight": "model-00001-of-00004.safetensors",
650
- "visual.blocks.31.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
651
- "visual.blocks.31.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
652
- "visual.blocks.31.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
653
- "visual.blocks.31.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
654
- "visual.blocks.31.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
655
- "visual.blocks.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
656
- "visual.blocks.31.norm1.weight": "model-00001-of-00004.safetensors",
657
- "visual.blocks.31.norm2.weight": "model-00001-of-00004.safetensors",
658
- "visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
659
- "visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
660
- "visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
661
- "visual.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
662
- "visual.blocks.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
663
- "visual.blocks.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
664
- "visual.blocks.4.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
665
- "visual.blocks.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
666
- "visual.blocks.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
667
- "visual.blocks.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
668
- "visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
669
- "visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
670
- "visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
671
- "visual.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
672
- "visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
673
- "visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
674
- "visual.blocks.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
675
- "visual.blocks.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
676
- "visual.blocks.5.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
677
- "visual.blocks.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
678
- "visual.blocks.5.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
679
- "visual.blocks.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
680
- "visual.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
681
- "visual.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
682
- "visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
683
- "visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
684
- "visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
685
- "visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
686
- "visual.blocks.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
687
- "visual.blocks.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
688
- "visual.blocks.6.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
689
- "visual.blocks.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
690
- "visual.blocks.6.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
691
- "visual.blocks.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
692
- "visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
693
- "visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
694
- "visual.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
695
- "visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
696
- "visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
697
- "visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
698
- "visual.blocks.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
699
- "visual.blocks.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
700
- "visual.blocks.7.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
701
- "visual.blocks.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
702
- "visual.blocks.7.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
703
- "visual.blocks.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
704
- "visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
705
- "visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
706
- "visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
707
- "visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
708
- "visual.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
709
- "visual.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
710
- "visual.blocks.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
711
- "visual.blocks.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
712
- "visual.blocks.8.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
713
- "visual.blocks.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
714
- "visual.blocks.8.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
715
- "visual.blocks.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
716
- "visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
717
- "visual.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
718
- "visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
719
- "visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
720
- "visual.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
721
- "visual.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
722
- "visual.blocks.9.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
723
- "visual.blocks.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
724
- "visual.blocks.9.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
725
- "visual.blocks.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
726
- "visual.blocks.9.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
727
- "visual.blocks.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
728
- "visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
729
- "visual.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
730
- "visual.merger.ln_q.weight": "model-00001-of-00004.safetensors",
731
- "visual.merger.mlp.0.bias": "model-00001-of-00004.safetensors",
732
- "visual.merger.mlp.0.weight": "model-00001-of-00004.safetensors",
733
- "visual.merger.mlp.2.bias": "model-00001-of-00004.safetensors",
734
- "visual.merger.mlp.2.weight": "model-00001-of-00004.safetensors",
735
- "visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors"
736
- }
737
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/preprocessor_config.json DELETED
@@ -1,37 +0,0 @@
1
- {
2
- "crop_size": null,
3
- "data_format": "channels_first",
4
- "default_to_square": true,
5
- "device": null,
6
- "disable_grouping": null,
7
- "do_center_crop": null,
8
- "do_convert_rgb": true,
9
- "do_normalize": true,
10
- "do_rescale": true,
11
- "do_resize": true,
12
- "image_mean": [
13
- 0.48145466,
14
- 0.4578275,
15
- 0.40821073
16
- ],
17
- "image_processor_type": "Qwen2VLImageProcessorFast",
18
- "image_std": [
19
- 0.26862954,
20
- 0.26130258,
21
- 0.27577711
22
- ],
23
- "input_data_format": null,
24
- "max_pixels": 12845056,
25
- "merge_size": 2,
26
- "min_pixels": 3136,
27
- "patch_size": 14,
28
- "processor_class": "Qwen2_5_VLProcessor",
29
- "resample": 3,
30
- "rescale_factor": 0.00392156862745098,
31
- "return_tensors": null,
32
- "size": {
33
- "longest_edge": 12845056,
34
- "shortest_edge": 3136
35
- },
36
- "temporal_patch_size": 2
37
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/rng_state_0.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8bfe1981024ef92f2da08a90c72c7c793d1cc9de1547abd2556c968be70232eb
3
- size 16389
 
 
 
 
size-80k/checkpoint-1200/rng_state_1.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a35b845d476d830805793c3dcf8ac2daad87fec289bff3f7eda9e72fc374eda1
3
- size 16389
 
 
 
 
size-80k/checkpoint-1200/rng_state_2.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:03e9880996b01262a807d1ec3ebd91eee540e08130a14a45a4648731fd0d48a9
3
- size 16389
 
 
 
 
size-80k/checkpoint-1200/rng_state_3.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ee25c237d6fe62ec76adcf7daf899d7ed32eab5d1a5b447b911f4451c9a1b258
3
- size 16389
 
 
 
 
size-80k/checkpoint-1200/rng_state_4.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7a6b31133f29a8fc0cb538aa807d6a403bd51939336bfd425cd3d122d8c5595c
3
- size 16389
 
 
 
 
size-80k/checkpoint-1200/rng_state_5.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a26c55b5c7fa0522b1d27b2c00a7ea77ad010f19a1321991165c5c972b8fa97a
3
- size 16389
 
 
 
 
size-80k/checkpoint-1200/rng_state_6.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4a1a3cf85626196804f25a8293e22dc561bba068a70fb123e04afe4896c33972
3
- size 16389
 
 
 
 
size-80k/checkpoint-1200/rng_state_7.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:28f87c1ee5f5db346c7b913137cbccd196eaf8ec5a4cf9f192418a3069269b49
3
- size 16389
 
 
 
 
size-80k/checkpoint-1200/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e57acaba9f6ad15a4c4e39411a0f1a3cbb759ce086b048cb24d41a87db48f21a
3
- size 1465
 
 
 
 
size-80k/checkpoint-1200/special_tokens_map.json DELETED
@@ -1,31 +0,0 @@
1
- {
2
- "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>",
5
- "<|object_ref_start|>",
6
- "<|object_ref_end|>",
7
- "<|box_start|>",
8
- "<|box_end|>",
9
- "<|quad_start|>",
10
- "<|quad_end|>",
11
- "<|vision_start|>",
12
- "<|vision_end|>",
13
- "<|vision_pad|>",
14
- "<|image_pad|>",
15
- "<|video_pad|>"
16
- ],
17
- "eos_token": {
18
- "content": "<|im_end|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- },
24
- "pad_token": {
25
- "content": "<|endoftext|>",
26
- "lstrip": false,
27
- "normalized": false,
28
- "rstrip": false,
29
- "single_word": false
30
- }
31
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4c9f7c087bb10192b30b4697b05fdaec59883791f4f6defca36d4e8d9891538e
3
- size 11422858
 
 
 
 
size-80k/checkpoint-1200/tokenizer_config.json DELETED
@@ -1,249 +0,0 @@
1
- {
2
- "add_bos_token": false,
3
- "add_prefix_space": false,
4
- "added_tokens_decoder": {
5
- "151643": {
6
- "content": "<|endoftext|>",
7
- "lstrip": false,
8
- "normalized": false,
9
- "rstrip": false,
10
- "single_word": false,
11
- "special": true
12
- },
13
- "151644": {
14
- "content": "<|im_start|>",
15
- "lstrip": false,
16
- "normalized": false,
17
- "rstrip": false,
18
- "single_word": false,
19
- "special": true
20
- },
21
- "151645": {
22
- "content": "<|im_end|>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false,
27
- "special": true
28
- },
29
- "151646": {
30
- "content": "<|object_ref_start|>",
31
- "lstrip": false,
32
- "normalized": false,
33
- "rstrip": false,
34
- "single_word": false,
35
- "special": true
36
- },
37
- "151647": {
38
- "content": "<|object_ref_end|>",
39
- "lstrip": false,
40
- "normalized": false,
41
- "rstrip": false,
42
- "single_word": false,
43
- "special": true
44
- },
45
- "151648": {
46
- "content": "<|box_start|>",
47
- "lstrip": false,
48
- "normalized": false,
49
- "rstrip": false,
50
- "single_word": false,
51
- "special": true
52
- },
53
- "151649": {
54
- "content": "<|box_end|>",
55
- "lstrip": false,
56
- "normalized": false,
57
- "rstrip": false,
58
- "single_word": false,
59
- "special": true
60
- },
61
- "151650": {
62
- "content": "<|quad_start|>",
63
- "lstrip": false,
64
- "normalized": false,
65
- "rstrip": false,
66
- "single_word": false,
67
- "special": true
68
- },
69
- "151651": {
70
- "content": "<|quad_end|>",
71
- "lstrip": false,
72
- "normalized": false,
73
- "rstrip": false,
74
- "single_word": false,
75
- "special": true
76
- },
77
- "151652": {
78
- "content": "<|vision_start|>",
79
- "lstrip": false,
80
- "normalized": false,
81
- "rstrip": false,
82
- "single_word": false,
83
- "special": true
84
- },
85
- "151653": {
86
- "content": "<|vision_end|>",
87
- "lstrip": false,
88
- "normalized": false,
89
- "rstrip": false,
90
- "single_word": false,
91
- "special": true
92
- },
93
- "151654": {
94
- "content": "<|vision_pad|>",
95
- "lstrip": false,
96
- "normalized": false,
97
- "rstrip": false,
98
- "single_word": false,
99
- "special": true
100
- },
101
- "151655": {
102
- "content": "<|image_pad|>",
103
- "lstrip": false,
104
- "normalized": false,
105
- "rstrip": false,
106
- "single_word": false,
107
- "special": true
108
- },
109
- "151656": {
110
- "content": "<|video_pad|>",
111
- "lstrip": false,
112
- "normalized": false,
113
- "rstrip": false,
114
- "single_word": false,
115
- "special": true
116
- },
117
- "151657": {
118
- "content": "<tool_call>",
119
- "lstrip": false,
120
- "normalized": false,
121
- "rstrip": false,
122
- "single_word": false,
123
- "special": false
124
- },
125
- "151658": {
126
- "content": "</tool_call>",
127
- "lstrip": false,
128
- "normalized": false,
129
- "rstrip": false,
130
- "single_word": false,
131
- "special": false
132
- },
133
- "151659": {
134
- "content": "<|fim_prefix|>",
135
- "lstrip": false,
136
- "normalized": false,
137
- "rstrip": false,
138
- "single_word": false,
139
- "special": false
140
- },
141
- "151660": {
142
- "content": "<|fim_middle|>",
143
- "lstrip": false,
144
- "normalized": false,
145
- "rstrip": false,
146
- "single_word": false,
147
- "special": false
148
- },
149
- "151661": {
150
- "content": "<|fim_suffix|>",
151
- "lstrip": false,
152
- "normalized": false,
153
- "rstrip": false,
154
- "single_word": false,
155
- "special": false
156
- },
157
- "151662": {
158
- "content": "<|fim_pad|>",
159
- "lstrip": false,
160
- "normalized": false,
161
- "rstrip": false,
162
- "single_word": false,
163
- "special": false
164
- },
165
- "151663": {
166
- "content": "<|repo_name|>",
167
- "lstrip": false,
168
- "normalized": false,
169
- "rstrip": false,
170
- "single_word": false,
171
- "special": false
172
- },
173
- "151664": {
174
- "content": "<|file_sep|>",
175
- "lstrip": false,
176
- "normalized": false,
177
- "rstrip": false,
178
- "single_word": false,
179
- "special": false
180
- },
181
- "151665": {
182
- "content": "<abs_vis_token_pad>",
183
- "lstrip": false,
184
- "normalized": false,
185
- "rstrip": false,
186
- "single_word": false,
187
- "special": true
188
- },
189
- "151666": {
190
- "content": "<abs_vis_token>",
191
- "lstrip": false,
192
- "normalized": false,
193
- "rstrip": false,
194
- "single_word": false,
195
- "special": true
196
- },
197
- "151667": {
198
- "content": "</abs_vis_token>",
199
- "lstrip": false,
200
- "normalized": false,
201
- "rstrip": false,
202
- "single_word": false,
203
- "special": true
204
- },
205
- "151668": {
206
- "content": "<observation>",
207
- "lstrip": false,
208
- "normalized": false,
209
- "rstrip": false,
210
- "single_word": false,
211
- "special": true
212
- },
213
- "151669": {
214
- "content": "</observation>",
215
- "lstrip": false,
216
- "normalized": false,
217
- "rstrip": false,
218
- "single_word": false,
219
- "special": true
220
- }
221
- },
222
- "additional_special_tokens": [
223
- "<|im_start|>",
224
- "<|im_end|>",
225
- "<|object_ref_start|>",
226
- "<|object_ref_end|>",
227
- "<|box_start|>",
228
- "<|box_end|>",
229
- "<|quad_start|>",
230
- "<|quad_end|>",
231
- "<|vision_start|>",
232
- "<|vision_end|>",
233
- "<|vision_pad|>",
234
- "<|image_pad|>",
235
- "<|video_pad|>"
236
- ],
237
- "bos_token": null,
238
- "clean_up_tokenization_spaces": false,
239
- "eos_token": "<|im_end|>",
240
- "errors": "replace",
241
- "extra_special_tokens": {},
242
- "model_max_length": 131072,
243
- "pad_token": "<|endoftext|>",
244
- "processor_class": "Qwen2_5_VLProcessor",
245
- "split_special_tokens": false,
246
- "tokenizer_class": "Qwen2Tokenizer",
247
- "unk_token": null,
248
- "use_fast": true
249
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/trainer_state.json DELETED
@@ -1,1354 +0,0 @@
1
- {
2
- "best_global_step": null,
3
- "best_metric": null,
4
- "best_model_checkpoint": null,
5
- "epoch": 1.92,
6
- "eval_steps": 500,
7
- "global_step": 1200,
8
- "is_hyper_param_search": false,
9
- "is_local_process_zero": true,
10
- "is_world_process_zero": true,
11
- "log_history": [
12
- {
13
- "alignment_loss": 0.862792,
14
- "epoch": 0.016,
15
- "grad_norm": 718.0257568359375,
16
- "learning_rate": 9e-06,
17
- "loss": 64.1373,
18
- "mean_token_accuracy": 0.6235283881425857,
19
- "num_tokens": 318050.0,
20
- "step": 10,
21
- "teacher_ce_loss": 29.904943
22
- },
23
- {
24
- "alignment_loss": 0.903638,
25
- "epoch": 0.032,
26
- "grad_norm": 364.2002868652344,
27
- "learning_rate": 9.927419354838711e-06,
28
- "loss": 18.8553,
29
- "mean_token_accuracy": 0.7676555842161179,
30
- "num_tokens": 633824.0,
31
- "step": 20,
32
- "teacher_ce_loss": 9.389782
33
- },
34
- {
35
- "alignment_loss": 0.860134,
36
- "epoch": 0.048,
37
- "grad_norm": 358.1412353515625,
38
- "learning_rate": 9.846774193548388e-06,
39
- "loss": 14.6034,
40
- "mean_token_accuracy": 0.7984995275735856,
41
- "num_tokens": 952442.0,
42
- "step": 30,
43
- "teacher_ce_loss": 6.745243
44
- },
45
- {
46
- "alignment_loss": 0.845411,
47
- "epoch": 0.064,
48
- "grad_norm": 258.8160400390625,
49
- "learning_rate": 9.766129032258065e-06,
50
- "loss": 10.0801,
51
- "mean_token_accuracy": 0.8730748683214188,
52
- "num_tokens": 1267476.0,
53
- "step": 40,
54
- "teacher_ce_loss": 5.405817
55
- },
56
- {
57
- "alignment_loss": 0.809125,
58
- "epoch": 0.08,
59
- "grad_norm": 74.79056549072266,
60
- "learning_rate": 9.685483870967743e-06,
61
- "loss": 8.1846,
62
- "mean_token_accuracy": 0.8569030702114105,
63
- "num_tokens": 1581570.0,
64
- "step": 50,
65
- "teacher_ce_loss": 4.068844
66
- },
67
- {
68
- "alignment_loss": 0.780713,
69
- "epoch": 0.096,
70
- "grad_norm": 65.06912231445312,
71
- "learning_rate": 9.60483870967742e-06,
72
- "loss": 7.224,
73
- "mean_token_accuracy": 0.8648413270711899,
74
- "num_tokens": 1893333.0,
75
- "step": 60,
76
- "teacher_ce_loss": 3.466415
77
- },
78
- {
79
- "alignment_loss": 0.780829,
80
- "epoch": 0.112,
81
- "grad_norm": 60.01629638671875,
82
- "learning_rate": 9.524193548387098e-06,
83
- "loss": 7.5994,
84
- "mean_token_accuracy": 0.8559162706136704,
85
- "num_tokens": 2209164.0,
86
- "step": 70,
87
- "teacher_ce_loss": 3.94546
88
- },
89
- {
90
- "alignment_loss": 0.734224,
91
- "epoch": 0.128,
92
- "grad_norm": 59.115631103515625,
93
- "learning_rate": 9.443548387096774e-06,
94
- "loss": 7.9446,
95
- "mean_token_accuracy": 0.8487120151519776,
96
- "num_tokens": 2522302.0,
97
- "step": 80,
98
- "teacher_ce_loss": 3.444979
99
- },
100
- {
101
- "alignment_loss": 0.728953,
102
- "epoch": 0.144,
103
- "grad_norm": 59.85231399536133,
104
- "learning_rate": 9.362903225806452e-06,
105
- "loss": 7.1798,
106
- "mean_token_accuracy": 0.8576172322034836,
107
- "num_tokens": 2840291.0,
108
- "step": 90,
109
- "teacher_ce_loss": 3.875605
110
- },
111
- {
112
- "alignment_loss": 0.705238,
113
- "epoch": 0.16,
114
- "grad_norm": 57.21866226196289,
115
- "learning_rate": 9.28225806451613e-06,
116
- "loss": 7.0143,
117
- "mean_token_accuracy": 0.8610904812812805,
118
- "num_tokens": 3153329.0,
119
- "step": 100,
120
- "teacher_ce_loss": 3.915993
121
- },
122
- {
123
- "alignment_loss": 0.65598,
124
- "epoch": 0.176,
125
- "grad_norm": 53.44105911254883,
126
- "learning_rate": 9.201612903225807e-06,
127
- "loss": 7.0262,
128
- "mean_token_accuracy": 0.8637181729078293,
129
- "num_tokens": 3470261.0,
130
- "step": 110,
131
- "teacher_ce_loss": 3.817101
132
- },
133
- {
134
- "alignment_loss": 0.644482,
135
- "epoch": 0.192,
136
- "grad_norm": 50.39668655395508,
137
- "learning_rate": 9.120967741935485e-06,
138
- "loss": 6.7324,
139
- "mean_token_accuracy": 0.8652872711420059,
140
- "num_tokens": 3788161.0,
141
- "step": 120,
142
- "teacher_ce_loss": 3.510738
143
- },
144
- {
145
- "alignment_loss": 0.649994,
146
- "epoch": 0.208,
147
- "grad_norm": 53.8228759765625,
148
- "learning_rate": 9.040322580645162e-06,
149
- "loss": 7.1052,
150
- "mean_token_accuracy": 0.8574498772621155,
151
- "num_tokens": 4103519.0,
152
- "step": 130,
153
- "teacher_ce_loss": 3.533933
154
- },
155
- {
156
- "alignment_loss": 0.656507,
157
- "epoch": 0.224,
158
- "grad_norm": 133.0367889404297,
159
- "learning_rate": 8.95967741935484e-06,
160
- "loss": 6.835,
161
- "mean_token_accuracy": 0.8623693853616714,
162
- "num_tokens": 4419625.0,
163
- "step": 140,
164
- "teacher_ce_loss": 3.452581
165
- },
166
- {
167
- "alignment_loss": 0.633311,
168
- "epoch": 0.24,
169
- "grad_norm": 50.68903350830078,
170
- "learning_rate": 8.879032258064517e-06,
171
- "loss": 6.6745,
172
- "mean_token_accuracy": 0.8657911330461502,
173
- "num_tokens": 4730991.0,
174
- "step": 150,
175
- "teacher_ce_loss": 3.536862
176
- },
177
- {
178
- "alignment_loss": 0.626141,
179
- "epoch": 0.256,
180
- "grad_norm": 51.91237258911133,
181
- "learning_rate": 8.798387096774195e-06,
182
- "loss": 6.6621,
183
- "mean_token_accuracy": 0.8711463153362274,
184
- "num_tokens": 5046078.0,
185
- "step": 160,
186
- "teacher_ce_loss": 3.239135
187
- },
188
- {
189
- "alignment_loss": 0.623654,
190
- "epoch": 0.272,
191
- "grad_norm": 55.68531799316406,
192
- "learning_rate": 8.717741935483872e-06,
193
- "loss": 7.105,
194
- "mean_token_accuracy": 0.8593070149421692,
195
- "num_tokens": 5360908.0,
196
- "step": 170,
197
- "teacher_ce_loss": 4.043244
198
- },
199
- {
200
- "alignment_loss": 0.59826,
201
- "epoch": 0.288,
202
- "grad_norm": 47.595821380615234,
203
- "learning_rate": 8.63709677419355e-06,
204
- "loss": 6.6381,
205
- "mean_token_accuracy": 0.8728302627801895,
206
- "num_tokens": 5676365.0,
207
- "step": 180,
208
- "teacher_ce_loss": 3.159675
209
- },
210
- {
211
- "alignment_loss": 0.592354,
212
- "epoch": 0.304,
213
- "grad_norm": 51.66691589355469,
214
- "learning_rate": 8.556451612903226e-06,
215
- "loss": 6.9669,
216
- "mean_token_accuracy": 0.8681543469429016,
217
- "num_tokens": 5989631.0,
218
- "step": 190,
219
- "teacher_ce_loss": 3.414008
220
- },
221
- {
222
- "alignment_loss": 0.591145,
223
- "epoch": 0.32,
224
- "grad_norm": 51.13317108154297,
225
- "learning_rate": 8.475806451612903e-06,
226
- "loss": 6.3797,
227
- "mean_token_accuracy": 0.8751179337501526,
228
- "num_tokens": 6305475.0,
229
- "step": 200,
230
- "teacher_ce_loss": 3.851559
231
- },
232
- {
233
- "alignment_loss": 0.555417,
234
- "epoch": 0.336,
235
- "grad_norm": 45.45359420776367,
236
- "learning_rate": 8.395161290322581e-06,
237
- "loss": 5.5984,
238
- "mean_token_accuracy": 0.8909706950187684,
239
- "num_tokens": 314036.0,
240
- "step": 210,
241
- "teacher_ce_loss": 3.146526
242
- },
243
- {
244
- "alignment_loss": 0.577421,
245
- "epoch": 0.352,
246
- "grad_norm": 40.985679626464844,
247
- "learning_rate": 8.314516129032258e-06,
248
- "loss": 5.6594,
249
- "mean_token_accuracy": 0.8904107630252838,
250
- "num_tokens": 628087.0,
251
- "step": 220,
252
- "teacher_ce_loss": 3.197752
253
- },
254
- {
255
- "alignment_loss": 0.56576,
256
- "epoch": 0.368,
257
- "grad_norm": 47.757164001464844,
258
- "learning_rate": 8.233870967741936e-06,
259
- "loss": 5.9419,
260
- "mean_token_accuracy": 0.8838070333003998,
261
- "num_tokens": 942974.0,
262
- "step": 230,
263
- "teacher_ce_loss": 2.854954
264
- },
265
- {
266
- "alignment_loss": 0.586017,
267
- "epoch": 0.384,
268
- "grad_norm": 44.95077133178711,
269
- "learning_rate": 8.153225806451614e-06,
270
- "loss": 6.0853,
271
- "mean_token_accuracy": 0.8805270850658417,
272
- "num_tokens": 1257174.0,
273
- "step": 240,
274
- "teacher_ce_loss": 3.22536
275
- },
276
- {
277
- "alignment_loss": 0.602747,
278
- "epoch": 0.4,
279
- "grad_norm": 44.999000549316406,
280
- "learning_rate": 8.07258064516129e-06,
281
- "loss": 6.0178,
282
- "mean_token_accuracy": 0.8793079227209091,
283
- "num_tokens": 1571795.0,
284
- "step": 250,
285
- "teacher_ce_loss": 4.098481
286
- },
287
- {
288
- "alignment_loss": 0.545827,
289
- "epoch": 0.416,
290
- "grad_norm": 47.33550262451172,
291
- "learning_rate": 7.991935483870969e-06,
292
- "loss": 6.5041,
293
- "mean_token_accuracy": 0.8660819709300995,
294
- "num_tokens": 1888481.0,
295
- "step": 260,
296
- "teacher_ce_loss": 3.163868
297
- },
298
- {
299
- "alignment_loss": 0.591056,
300
- "epoch": 0.432,
301
- "grad_norm": 44.00401306152344,
302
- "learning_rate": 7.911290322580646e-06,
303
- "loss": 5.8824,
304
- "mean_token_accuracy": 0.8814180880784989,
305
- "num_tokens": 2204965.0,
306
- "step": 270,
307
- "teacher_ce_loss": 3.135252
308
- },
309
- {
310
- "alignment_loss": 0.570753,
311
- "epoch": 0.448,
312
- "grad_norm": 44.05668640136719,
313
- "learning_rate": 7.830645161290324e-06,
314
- "loss": 6.1337,
315
- "mean_token_accuracy": 0.8784295946359635,
316
- "num_tokens": 2516325.0,
317
- "step": 280,
318
- "teacher_ce_loss": 3.090131
319
- },
320
- {
321
- "alignment_loss": 0.514921,
322
- "epoch": 0.464,
323
- "grad_norm": 55.573421478271484,
324
- "learning_rate": 7.75e-06,
325
- "loss": 6.283,
326
- "mean_token_accuracy": 0.8741924107074738,
327
- "num_tokens": 2835148.0,
328
- "step": 290,
329
- "teacher_ce_loss": 3.036129
330
- },
331
- {
332
- "alignment_loss": 0.566925,
333
- "epoch": 0.48,
334
- "grad_norm": 57.75579833984375,
335
- "learning_rate": 7.669354838709679e-06,
336
- "loss": 6.0555,
337
- "mean_token_accuracy": 0.8747823536396027,
338
- "num_tokens": 3150924.0,
339
- "step": 300,
340
- "teacher_ce_loss": 3.202478
341
- },
342
- {
343
- "alignment_loss": 0.564667,
344
- "epoch": 0.496,
345
- "grad_norm": 49.43717575073242,
346
- "learning_rate": 7.588709677419356e-06,
347
- "loss": 5.9669,
348
- "mean_token_accuracy": 0.8815405040979385,
349
- "num_tokens": 3470779.0,
350
- "step": 310,
351
- "teacher_ce_loss": 3.208461
352
- },
353
- {
354
- "alignment_loss": 0.529164,
355
- "epoch": 0.512,
356
- "grad_norm": 46.63803482055664,
357
- "learning_rate": 7.508064516129033e-06,
358
- "loss": 6.3799,
359
- "mean_token_accuracy": 0.8730729401111603,
360
- "num_tokens": 3783441.0,
361
- "step": 320,
362
- "teacher_ce_loss": 3.5823
363
- },
364
- {
365
- "alignment_loss": 0.563581,
366
- "epoch": 0.528,
367
- "grad_norm": 46.183712005615234,
368
- "learning_rate": 7.427419354838711e-06,
369
- "loss": 6.2433,
370
- "mean_token_accuracy": 0.8735756903886795,
371
- "num_tokens": 4101059.0,
372
- "step": 330,
373
- "teacher_ce_loss": 3.483996
374
- },
375
- {
376
- "alignment_loss": 0.546565,
377
- "epoch": 0.544,
378
- "grad_norm": 44.92106628417969,
379
- "learning_rate": 7.346774193548387e-06,
380
- "loss": 5.8321,
381
- "mean_token_accuracy": 0.8890390068292617,
382
- "num_tokens": 4416413.0,
383
- "step": 340,
384
- "teacher_ce_loss": 2.801952
385
- },
386
- {
387
- "alignment_loss": 0.55359,
388
- "epoch": 0.56,
389
- "grad_norm": 46.407737731933594,
390
- "learning_rate": 7.266129032258065e-06,
391
- "loss": 5.87,
392
- "mean_token_accuracy": 0.8832425236701965,
393
- "num_tokens": 4730003.0,
394
- "step": 350,
395
- "teacher_ce_loss": 2.549353
396
- },
397
- {
398
- "alignment_loss": 0.54398,
399
- "epoch": 0.576,
400
- "grad_norm": 44.54058837890625,
401
- "learning_rate": 7.185483870967742e-06,
402
- "loss": 5.897,
403
- "mean_token_accuracy": 0.8831360638141632,
404
- "num_tokens": 5043331.0,
405
- "step": 360,
406
- "teacher_ce_loss": 3.334575
407
- },
408
- {
409
- "alignment_loss": 0.554904,
410
- "epoch": 0.592,
411
- "grad_norm": 40.729427337646484,
412
- "learning_rate": 7.1048387096774195e-06,
413
- "loss": 5.7639,
414
- "mean_token_accuracy": 0.8826215326786041,
415
- "num_tokens": 5358483.0,
416
- "step": 370,
417
- "teacher_ce_loss": 2.525457
418
- },
419
- {
420
- "alignment_loss": 0.554032,
421
- "epoch": 0.608,
422
- "grad_norm": 46.933616638183594,
423
- "learning_rate": 7.024193548387097e-06,
424
- "loss": 5.7075,
425
- "mean_token_accuracy": 0.8849380105733872,
426
- "num_tokens": 5673525.0,
427
- "step": 380,
428
- "teacher_ce_loss": 3.37282
429
- },
430
- {
431
- "alignment_loss": 0.534627,
432
- "epoch": 0.624,
433
- "grad_norm": 54.1641960144043,
434
- "learning_rate": 6.943548387096774e-06,
435
- "loss": 5.5729,
436
- "mean_token_accuracy": 0.8884122371673584,
437
- "num_tokens": 5987505.0,
438
- "step": 390,
439
- "teacher_ce_loss": 3.009932
440
- },
441
- {
442
- "alignment_loss": 0.546374,
443
- "epoch": 0.64,
444
- "grad_norm": 42.544464111328125,
445
- "learning_rate": 6.862903225806452e-06,
446
- "loss": 5.5001,
447
- "mean_token_accuracy": 0.8928223967552185,
448
- "num_tokens": 6300985.0,
449
- "step": 400,
450
- "teacher_ce_loss": 2.863962
451
- },
452
- {
453
- "alignment_loss": 0.542254,
454
- "epoch": 0.656,
455
- "grad_norm": 44.9060173034668,
456
- "learning_rate": 6.78225806451613e-06,
457
- "loss": 5.6526,
458
- "mean_token_accuracy": 0.8842327892780304,
459
- "num_tokens": 6617569.0,
460
- "step": 410,
461
- "teacher_ce_loss": 2.375985
462
- },
463
- {
464
- "alignment_loss": 0.55457,
465
- "epoch": 0.672,
466
- "grad_norm": 44.16085433959961,
467
- "learning_rate": 6.701612903225807e-06,
468
- "loss": 5.6606,
469
- "mean_token_accuracy": 0.8866721272468567,
470
- "num_tokens": 6932763.0,
471
- "step": 420,
472
- "teacher_ce_loss": 2.443974
473
- },
474
- {
475
- "alignment_loss": 0.552511,
476
- "epoch": 0.688,
477
- "grad_norm": 51.63603973388672,
478
- "learning_rate": 6.620967741935485e-06,
479
- "loss": 5.8808,
480
- "mean_token_accuracy": 0.8786897659301758,
481
- "num_tokens": 7249623.0,
482
- "step": 430,
483
- "teacher_ce_loss": 2.891178
484
- },
485
- {
486
- "alignment_loss": 0.524473,
487
- "epoch": 0.704,
488
- "grad_norm": 46.281097412109375,
489
- "learning_rate": 6.540322580645162e-06,
490
- "loss": 5.7264,
491
- "mean_token_accuracy": 0.8836472988128662,
492
- "num_tokens": 7564269.0,
493
- "step": 440,
494
- "teacher_ce_loss": 2.679617
495
- },
496
- {
497
- "alignment_loss": 0.529236,
498
- "epoch": 0.72,
499
- "grad_norm": 44.076759338378906,
500
- "learning_rate": 6.4596774193548396e-06,
501
- "loss": 5.3558,
502
- "mean_token_accuracy": 0.8936043500900268,
503
- "num_tokens": 7880174.0,
504
- "step": 450,
505
- "teacher_ce_loss": 3.162974
506
- },
507
- {
508
- "alignment_loss": 0.549703,
509
- "epoch": 0.736,
510
- "grad_norm": 40.00216293334961,
511
- "learning_rate": 6.379032258064517e-06,
512
- "loss": 6.2029,
513
- "mean_token_accuracy": 0.8711945086717605,
514
- "num_tokens": 8194286.0,
515
- "step": 460,
516
- "teacher_ce_loss": 3.744506
517
- },
518
- {
519
- "alignment_loss": 0.536025,
520
- "epoch": 0.752,
521
- "grad_norm": 44.7592887878418,
522
- "learning_rate": 6.298387096774194e-06,
523
- "loss": 6.032,
524
- "mean_token_accuracy": 0.8738157570362091,
525
- "num_tokens": 8510188.0,
526
- "step": 470,
527
- "teacher_ce_loss": 2.938508
528
- },
529
- {
530
- "alignment_loss": 0.550875,
531
- "epoch": 0.768,
532
- "grad_norm": 46.17378234863281,
533
- "learning_rate": 6.217741935483872e-06,
534
- "loss": 6.2938,
535
- "mean_token_accuracy": 0.8732119083404541,
536
- "num_tokens": 8823232.0,
537
- "step": 480,
538
- "teacher_ce_loss": 3.416612
539
- },
540
- {
541
- "alignment_loss": 0.544042,
542
- "epoch": 0.784,
543
- "grad_norm": 41.7293815612793,
544
- "learning_rate": 6.137096774193549e-06,
545
- "loss": 5.8207,
546
- "mean_token_accuracy": 0.8850664228200913,
547
- "num_tokens": 9137164.0,
548
- "step": 490,
549
- "teacher_ce_loss": 2.889406
550
- },
551
- {
552
- "alignment_loss": 0.523621,
553
- "epoch": 0.8,
554
- "grad_norm": 45.666717529296875,
555
- "learning_rate": 6.056451612903226e-06,
556
- "loss": 5.6623,
557
- "mean_token_accuracy": 0.8847638428211212,
558
- "num_tokens": 9454007.0,
559
- "step": 500,
560
- "teacher_ce_loss": 2.617646
561
- },
562
- {
563
- "alignment_loss": 0.543883,
564
- "epoch": 0.816,
565
- "grad_norm": 42.79743957519531,
566
- "learning_rate": 5.975806451612903e-06,
567
- "loss": 5.6651,
568
- "mean_token_accuracy": 0.882024610042572,
569
- "num_tokens": 9770217.0,
570
- "step": 510,
571
- "teacher_ce_loss": 2.62713
572
- },
573
- {
574
- "alignment_loss": 0.516073,
575
- "epoch": 0.832,
576
- "grad_norm": 46.972843170166016,
577
- "learning_rate": 5.8951612903225805e-06,
578
- "loss": 6.1587,
579
- "mean_token_accuracy": 0.8783568263053894,
580
- "num_tokens": 10084137.0,
581
- "step": 520,
582
- "teacher_ce_loss": 2.998152
583
- },
584
- {
585
- "alignment_loss": 0.54162,
586
- "epoch": 0.848,
587
- "grad_norm": 43.42284393310547,
588
- "learning_rate": 5.814516129032258e-06,
589
- "loss": 5.8265,
590
- "mean_token_accuracy": 0.8861030846834183,
591
- "num_tokens": 10398901.0,
592
- "step": 530,
593
- "teacher_ce_loss": 3.187277
594
- },
595
- {
596
- "alignment_loss": 0.538004,
597
- "epoch": 0.864,
598
- "grad_norm": 44.71125030517578,
599
- "learning_rate": 5.733870967741936e-06,
600
- "loss": 5.7322,
601
- "mean_token_accuracy": 0.8832547843456269,
602
- "num_tokens": 10711856.0,
603
- "step": 540,
604
- "teacher_ce_loss": 2.750253
605
- },
606
- {
607
- "alignment_loss": 0.541207,
608
- "epoch": 0.88,
609
- "grad_norm": 43.257076263427734,
610
- "learning_rate": 5.6532258064516136e-06,
611
- "loss": 5.4765,
612
- "mean_token_accuracy": 0.886500871181488,
613
- "num_tokens": 11029626.0,
614
- "step": 550,
615
- "teacher_ce_loss": 3.240631
616
- },
617
- {
618
- "alignment_loss": 0.543185,
619
- "epoch": 0.896,
620
- "grad_norm": 47.65825653076172,
621
- "learning_rate": 5.572580645161291e-06,
622
- "loss": 5.7465,
623
- "mean_token_accuracy": 0.8831908583641053,
624
- "num_tokens": 11346432.0,
625
- "step": 560,
626
- "teacher_ce_loss": 3.292619
627
- },
628
- {
629
- "alignment_loss": 0.548355,
630
- "epoch": 0.912,
631
- "grad_norm": 46.56586456298828,
632
- "learning_rate": 5.491935483870968e-06,
633
- "loss": 5.2755,
634
- "mean_token_accuracy": 0.8919913023710251,
635
- "num_tokens": 11659406.0,
636
- "step": 570,
637
- "teacher_ce_loss": 2.91626
638
- },
639
- {
640
- "alignment_loss": 0.52101,
641
- "epoch": 0.928,
642
- "grad_norm": 45.433738708496094,
643
- "learning_rate": 5.411290322580646e-06,
644
- "loss": 5.5511,
645
- "mean_token_accuracy": 0.890060544013977,
646
- "num_tokens": 11972782.0,
647
- "step": 580,
648
- "teacher_ce_loss": 3.449245
649
- },
650
- {
651
- "alignment_loss": 0.540634,
652
- "epoch": 0.944,
653
- "grad_norm": 40.90420150756836,
654
- "learning_rate": 5.330645161290323e-06,
655
- "loss": 5.0252,
656
- "mean_token_accuracy": 0.8989457249641418,
657
- "num_tokens": 12287150.0,
658
- "step": 590,
659
- "teacher_ce_loss": 2.424076
660
- },
661
- {
662
- "alignment_loss": 0.539022,
663
- "epoch": 0.96,
664
- "grad_norm": 47.69533157348633,
665
- "learning_rate": 5.2500000000000006e-06,
666
- "loss": 5.9203,
667
- "mean_token_accuracy": 0.8797043979167938,
668
- "num_tokens": 12605091.0,
669
- "step": 600,
670
- "teacher_ce_loss": 3.361504
671
- },
672
- {
673
- "alignment_loss": 0.543596,
674
- "epoch": 0.976,
675
- "grad_norm": 54.82185363769531,
676
- "learning_rate": 5.169354838709678e-06,
677
- "loss": 5.7646,
678
- "mean_token_accuracy": 0.8838818430900574,
679
- "num_tokens": 12915776.0,
680
- "step": 610,
681
- "teacher_ce_loss": 2.738911
682
- },
683
- {
684
- "alignment_loss": 0.552586,
685
- "epoch": 0.992,
686
- "grad_norm": 41.07416915893555,
687
- "learning_rate": 5.088709677419355e-06,
688
- "loss": 5.2023,
689
- "mean_token_accuracy": 0.8941108345985412,
690
- "num_tokens": 13229351.0,
691
- "step": 620,
692
- "teacher_ce_loss": 3.055145
693
- },
694
- {
695
- "alignment_loss": 0.54879,
696
- "epoch": 1.008,
697
- "grad_norm": 36.733577728271484,
698
- "learning_rate": 5.008064516129033e-06,
699
- "loss": 5.3079,
700
- "mean_token_accuracy": 0.891439464688301,
701
- "num_tokens": 13543363.0,
702
- "step": 630,
703
- "teacher_ce_loss": 2.776687
704
- },
705
- {
706
- "alignment_loss": 0.509064,
707
- "epoch": 1.024,
708
- "grad_norm": 36.02306365966797,
709
- "learning_rate": 4.92741935483871e-06,
710
- "loss": 3.8017,
711
- "mean_token_accuracy": 0.9194183051586151,
712
- "num_tokens": 13859066.0,
713
- "step": 640,
714
- "teacher_ce_loss": 1.666513
715
- },
716
- {
717
- "alignment_loss": 0.52774,
718
- "epoch": 1.04,
719
- "grad_norm": 37.0902214050293,
720
- "learning_rate": 4.8467741935483876e-06,
721
- "loss": 3.7555,
722
- "mean_token_accuracy": 0.9190860956907272,
723
- "num_tokens": 14176396.0,
724
- "step": 650,
725
- "teacher_ce_loss": 1.80784
726
- },
727
- {
728
- "alignment_loss": 0.511381,
729
- "epoch": 1.056,
730
- "grad_norm": 41.1185417175293,
731
- "learning_rate": 4.766129032258065e-06,
732
- "loss": 3.6184,
733
- "mean_token_accuracy": 0.9257269024848938,
734
- "num_tokens": 14490053.0,
735
- "step": 660,
736
- "teacher_ce_loss": 2.084512
737
- },
738
- {
739
- "alignment_loss": 0.532751,
740
- "epoch": 1.072,
741
- "grad_norm": 39.66856384277344,
742
- "learning_rate": 4.685483870967742e-06,
743
- "loss": 3.5673,
744
- "mean_token_accuracy": 0.9240388512611389,
745
- "num_tokens": 14800225.0,
746
- "step": 670,
747
- "teacher_ce_loss": 1.852832
748
- },
749
- {
750
- "alignment_loss": 0.518106,
751
- "epoch": 1.088,
752
- "grad_norm": 38.82719802856445,
753
- "learning_rate": 4.60483870967742e-06,
754
- "loss": 3.539,
755
- "mean_token_accuracy": 0.9250211596488953,
756
- "num_tokens": 15116259.0,
757
- "step": 680,
758
- "teacher_ce_loss": 2.258169
759
- },
760
- {
761
- "alignment_loss": 0.552023,
762
- "epoch": 1.104,
763
- "grad_norm": 44.993797302246094,
764
- "learning_rate": 4.524193548387097e-06,
765
- "loss": 3.7047,
766
- "mean_token_accuracy": 0.9203028440475464,
767
- "num_tokens": 15431673.0,
768
- "step": 690,
769
- "teacher_ce_loss": 1.886494
770
- },
771
- {
772
- "alignment_loss": 0.534711,
773
- "epoch": 1.12,
774
- "grad_norm": 36.35969924926758,
775
- "learning_rate": 4.4435483870967745e-06,
776
- "loss": 3.9765,
777
- "mean_token_accuracy": 0.9167239487171173,
778
- "num_tokens": 15749338.0,
779
- "step": 700,
780
- "teacher_ce_loss": 1.940746
781
- },
782
- {
783
- "alignment_loss": 0.552155,
784
- "epoch": 1.1360000000000001,
785
- "grad_norm": 41.56093215942383,
786
- "learning_rate": 4.362903225806452e-06,
787
- "loss": 3.6335,
788
- "mean_token_accuracy": 0.9237402349710464,
789
- "num_tokens": 16062079.0,
790
- "step": 710,
791
- "teacher_ce_loss": 1.811384
792
- },
793
- {
794
- "alignment_loss": 0.543712,
795
- "epoch": 1.152,
796
- "grad_norm": 45.057186126708984,
797
- "learning_rate": 4.282258064516129e-06,
798
- "loss": 3.8143,
799
- "mean_token_accuracy": 0.9191823929548264,
800
- "num_tokens": 16377424.0,
801
- "step": 720,
802
- "teacher_ce_loss": 1.992962
803
- },
804
- {
805
- "alignment_loss": 0.532968,
806
- "epoch": 1.168,
807
- "grad_norm": 42.391929626464844,
808
- "learning_rate": 4.201612903225807e-06,
809
- "loss": 3.7505,
810
- "mean_token_accuracy": 0.919136518239975,
811
- "num_tokens": 16691433.0,
812
- "step": 730,
813
- "teacher_ce_loss": 2.034134
814
- },
815
- {
816
- "alignment_loss": 0.548623,
817
- "epoch": 1.184,
818
- "grad_norm": 41.2871208190918,
819
- "learning_rate": 4.120967741935484e-06,
820
- "loss": 3.6542,
821
- "mean_token_accuracy": 0.9259105503559113,
822
- "num_tokens": 17005984.0,
823
- "step": 740,
824
- "teacher_ce_loss": 1.540412
825
- },
826
- {
827
- "alignment_loss": 0.509535,
828
- "epoch": 1.2,
829
- "grad_norm": 42.47368240356445,
830
- "learning_rate": 4.0403225806451615e-06,
831
- "loss": 3.7454,
832
- "mean_token_accuracy": 0.9214767009019852,
833
- "num_tokens": 17322710.0,
834
- "step": 750,
835
- "teacher_ce_loss": 2.08513
836
- },
837
- {
838
- "alignment_loss": 0.519729,
839
- "epoch": 1.216,
840
- "grad_norm": 35.216556549072266,
841
- "learning_rate": 3.959677419354839e-06,
842
- "loss": 3.6702,
843
- "mean_token_accuracy": 0.9275216341018677,
844
- "num_tokens": 17636673.0,
845
- "step": 760,
846
- "teacher_ce_loss": 1.730667
847
- },
848
- {
849
- "alignment_loss": 0.540075,
850
- "epoch": 1.232,
851
- "grad_norm": 40.935184478759766,
852
- "learning_rate": 3.879032258064516e-06,
853
- "loss": 3.8348,
854
- "mean_token_accuracy": 0.9185233175754547,
855
- "num_tokens": 17952228.0,
856
- "step": 770,
857
- "teacher_ce_loss": 2.02698
858
- },
859
- {
860
- "alignment_loss": 0.549642,
861
- "epoch": 1.248,
862
- "grad_norm": 43.087684631347656,
863
- "learning_rate": 3.7983870967741937e-06,
864
- "loss": 3.6601,
865
- "mean_token_accuracy": 0.9229311019182205,
866
- "num_tokens": 18267149.0,
867
- "step": 780,
868
- "teacher_ce_loss": 2.513444
869
- },
870
- {
871
- "alignment_loss": 0.510661,
872
- "epoch": 1.264,
873
- "grad_norm": 40.932735443115234,
874
- "learning_rate": 3.717741935483871e-06,
875
- "loss": 4.0123,
876
- "mean_token_accuracy": 0.9158858686685563,
877
- "num_tokens": 18582546.0,
878
- "step": 790,
879
- "teacher_ce_loss": 2.016326
880
- },
881
- {
882
- "alignment_loss": 0.543682,
883
- "epoch": 1.28,
884
- "grad_norm": 38.4617919921875,
885
- "learning_rate": 3.6370967741935485e-06,
886
- "loss": 3.536,
887
- "mean_token_accuracy": 0.9244677364826203,
888
- "num_tokens": 18899255.0,
889
- "step": 800,
890
- "teacher_ce_loss": 1.306068
891
- },
892
- {
893
- "alignment_loss": 0.548868,
894
- "epoch": 1.296,
895
- "grad_norm": 38.674964904785156,
896
- "learning_rate": 3.5564516129032264e-06,
897
- "loss": 4.0295,
898
- "mean_token_accuracy": 0.9117148578166961,
899
- "num_tokens": 19212787.0,
900
- "step": 810,
901
- "teacher_ce_loss": 1.864514
902
- },
903
- {
904
- "alignment_loss": 0.523737,
905
- "epoch": 1.312,
906
- "grad_norm": 38.545196533203125,
907
- "learning_rate": 3.4758064516129038e-06,
908
- "loss": 3.8058,
909
- "mean_token_accuracy": 0.9229878187179565,
910
- "num_tokens": 19526695.0,
911
- "step": 820,
912
- "teacher_ce_loss": 2.229923
913
- },
914
- {
915
- "alignment_loss": 0.491939,
916
- "epoch": 1.328,
917
- "grad_norm": 35.85627365112305,
918
- "learning_rate": 3.395161290322581e-06,
919
- "loss": 3.6742,
920
- "mean_token_accuracy": 0.9257246404886246,
921
- "num_tokens": 19844801.0,
922
- "step": 830,
923
- "teacher_ce_loss": 1.872186
924
- },
925
- {
926
- "alignment_loss": 0.535385,
927
- "epoch": 1.3439999999999999,
928
- "grad_norm": 41.145938873291016,
929
- "learning_rate": 3.3145161290322586e-06,
930
- "loss": 3.7644,
931
- "mean_token_accuracy": 0.9210176885128021,
932
- "num_tokens": 20158375.0,
933
- "step": 840,
934
- "teacher_ce_loss": 2.072073
935
- },
936
- {
937
- "alignment_loss": 0.52681,
938
- "epoch": 1.3599999999999999,
939
- "grad_norm": 41.83457946777344,
940
- "learning_rate": 3.2338709677419355e-06,
941
- "loss": 3.3073,
942
- "mean_token_accuracy": 0.928924959897995,
943
- "num_tokens": 20476226.0,
944
- "step": 850,
945
- "teacher_ce_loss": 1.963373
946
- },
947
- {
948
- "alignment_loss": 0.54648,
949
- "epoch": 1.376,
950
- "grad_norm": 42.46304702758789,
951
- "learning_rate": 3.153225806451613e-06,
952
- "loss": 3.8573,
953
- "mean_token_accuracy": 0.9205310791730881,
954
- "num_tokens": 20790341.0,
955
- "step": 860,
956
- "teacher_ce_loss": 1.995422
957
- },
958
- {
959
- "alignment_loss": 0.523628,
960
- "epoch": 1.392,
961
- "grad_norm": 44.286766052246094,
962
- "learning_rate": 3.0725806451612903e-06,
963
- "loss": 3.8721,
964
- "mean_token_accuracy": 0.9160841673612594,
965
- "num_tokens": 21103077.0,
966
- "step": 870,
967
- "teacher_ce_loss": 1.49945
968
- },
969
- {
970
- "alignment_loss": 0.527194,
971
- "epoch": 1.408,
972
- "grad_norm": 42.17543029785156,
973
- "learning_rate": 2.991935483870968e-06,
974
- "loss": 3.6655,
975
- "mean_token_accuracy": 0.923399806022644,
976
- "num_tokens": 21419870.0,
977
- "step": 880,
978
- "teacher_ce_loss": 1.914823
979
- },
980
- {
981
- "alignment_loss": 0.525613,
982
- "epoch": 1.424,
983
- "grad_norm": 43.81401824951172,
984
- "learning_rate": 2.9112903225806456e-06,
985
- "loss": 3.8179,
986
- "mean_token_accuracy": 0.9204582303762436,
987
- "num_tokens": 21736307.0,
988
- "step": 890,
989
- "teacher_ce_loss": 2.032509
990
- },
991
- {
992
- "alignment_loss": 0.552205,
993
- "epoch": 1.44,
994
- "grad_norm": 37.923824310302734,
995
- "learning_rate": 2.830645161290323e-06,
996
- "loss": 3.7228,
997
- "mean_token_accuracy": 0.9217585295438766,
998
- "num_tokens": 22048500.0,
999
- "step": 900,
1000
- "teacher_ce_loss": 2.064794
1001
- },
1002
- {
1003
- "alignment_loss": 0.52349,
1004
- "epoch": 1.456,
1005
- "grad_norm": 40.057708740234375,
1006
- "learning_rate": 2.7500000000000004e-06,
1007
- "loss": 3.9187,
1008
- "mean_token_accuracy": 0.917030531167984,
1009
- "num_tokens": 22365675.0,
1010
- "step": 910,
1011
- "teacher_ce_loss": 2.04531
1012
- },
1013
- {
1014
- "alignment_loss": 0.543985,
1015
- "epoch": 1.472,
1016
- "grad_norm": 41.5903205871582,
1017
- "learning_rate": 2.6693548387096773e-06,
1018
- "loss": 3.7059,
1019
- "mean_token_accuracy": 0.91900355219841,
1020
- "num_tokens": 22682280.0,
1021
- "step": 920,
1022
- "teacher_ce_loss": 2.253458
1023
- },
1024
- {
1025
- "alignment_loss": 0.542001,
1026
- "epoch": 1.488,
1027
- "grad_norm": 40.20334243774414,
1028
- "learning_rate": 2.5887096774193547e-06,
1029
- "loss": 3.6661,
1030
- "mean_token_accuracy": 0.9237508833408355,
1031
- "num_tokens": 22997316.0,
1032
- "step": 930,
1033
- "teacher_ce_loss": 1.596192
1034
- },
1035
- {
1036
- "alignment_loss": 0.510092,
1037
- "epoch": 1.504,
1038
- "grad_norm": 39.320125579833984,
1039
- "learning_rate": 2.5080645161290325e-06,
1040
- "loss": 3.5841,
1041
- "mean_token_accuracy": 0.9241526782512665,
1042
- "num_tokens": 23311902.0,
1043
- "step": 940,
1044
- "teacher_ce_loss": 1.993438
1045
- },
1046
- {
1047
- "alignment_loss": 0.513295,
1048
- "epoch": 1.52,
1049
- "grad_norm": 40.23579788208008,
1050
- "learning_rate": 2.42741935483871e-06,
1051
- "loss": 3.6999,
1052
- "mean_token_accuracy": 0.9233695477247238,
1053
- "num_tokens": 23624469.0,
1054
- "step": 950,
1055
- "teacher_ce_loss": 1.829836
1056
- },
1057
- {
1058
- "alignment_loss": 0.517973,
1059
- "epoch": 1.536,
1060
- "grad_norm": 42.18544387817383,
1061
- "learning_rate": 2.3467741935483873e-06,
1062
- "loss": 3.8719,
1063
- "mean_token_accuracy": 0.9172720789909363,
1064
- "num_tokens": 23939394.0,
1065
- "step": 960,
1066
- "teacher_ce_loss": 1.992009
1067
- },
1068
- {
1069
- "alignment_loss": 0.519922,
1070
- "epoch": 1.552,
1071
- "grad_norm": 36.45695495605469,
1072
- "learning_rate": 2.2661290322580647e-06,
1073
- "loss": 3.7144,
1074
- "mean_token_accuracy": 0.9190810173749924,
1075
- "num_tokens": 24253774.0,
1076
- "step": 970,
1077
- "teacher_ce_loss": 1.955374
1078
- },
1079
- {
1080
- "alignment_loss": 0.518951,
1081
- "epoch": 1.568,
1082
- "grad_norm": 40.88084411621094,
1083
- "learning_rate": 2.185483870967742e-06,
1084
- "loss": 3.7885,
1085
- "mean_token_accuracy": 0.9201515406370163,
1086
- "num_tokens": 24569229.0,
1087
- "step": 980,
1088
- "teacher_ce_loss": 2.108701
1089
- },
1090
- {
1091
- "alignment_loss": 0.52161,
1092
- "epoch": 1.584,
1093
- "grad_norm": 40.22114944458008,
1094
- "learning_rate": 2.1048387096774195e-06,
1095
- "loss": 3.9444,
1096
- "mean_token_accuracy": 0.9168345510959626,
1097
- "num_tokens": 24883140.0,
1098
- "step": 990,
1099
- "teacher_ce_loss": 2.466592
1100
- },
1101
- {
1102
- "alignment_loss": 0.537731,
1103
- "epoch": 1.6,
1104
- "grad_norm": 44.35193634033203,
1105
- "learning_rate": 2.024193548387097e-06,
1106
- "loss": 3.5547,
1107
- "mean_token_accuracy": 0.9260744124650955,
1108
- "num_tokens": 25200345.0,
1109
- "step": 1000,
1110
- "teacher_ce_loss": 1.864041
1111
- },
1112
- {
1113
- "alignment_loss": 0.52536,
1114
- "epoch": 1.616,
1115
- "grad_norm": 40.62210464477539,
1116
- "learning_rate": 1.9435483870967743e-06,
1117
- "loss": 3.4482,
1118
- "mean_token_accuracy": 0.9279706567525864,
1119
- "num_tokens": 25514363.0,
1120
- "step": 1010,
1121
- "teacher_ce_loss": 1.553494
1122
- },
1123
- {
1124
- "alignment_loss": 0.524492,
1125
- "epoch": 1.6320000000000001,
1126
- "grad_norm": 43.18842697143555,
1127
- "learning_rate": 1.8629032258064517e-06,
1128
- "loss": 3.388,
1129
- "mean_token_accuracy": 0.9309078633785248,
1130
- "num_tokens": 25828574.0,
1131
- "step": 1020,
1132
- "teacher_ce_loss": 1.580232
1133
- },
1134
- {
1135
- "alignment_loss": 0.530717,
1136
- "epoch": 1.6480000000000001,
1137
- "grad_norm": 43.79351806640625,
1138
- "learning_rate": 1.7822580645161291e-06,
1139
- "loss": 3.2208,
1140
- "mean_token_accuracy": 0.9347890436649322,
1141
- "num_tokens": 26143471.0,
1142
- "step": 1030,
1143
- "teacher_ce_loss": 1.50971
1144
- },
1145
- {
1146
- "alignment_loss": 0.52153,
1147
- "epoch": 1.6640000000000001,
1148
- "grad_norm": 34.18009948730469,
1149
- "learning_rate": 1.7016129032258065e-06,
1150
- "loss": 3.9054,
1151
- "mean_token_accuracy": 0.9174640119075775,
1152
- "num_tokens": 26458181.0,
1153
- "step": 1040,
1154
- "teacher_ce_loss": 2.483193
1155
- },
1156
- {
1157
- "alignment_loss": 0.529254,
1158
- "epoch": 1.6800000000000002,
1159
- "grad_norm": 43.49956512451172,
1160
- "learning_rate": 1.6209677419354842e-06,
1161
- "loss": 3.9325,
1162
- "mean_token_accuracy": 0.9165703773498535,
1163
- "num_tokens": 26771625.0,
1164
- "step": 1050,
1165
- "teacher_ce_loss": 2.772699
1166
- },
1167
- {
1168
- "alignment_loss": 0.517759,
1169
- "epoch": 1.696,
1170
- "grad_norm": 44.48139572143555,
1171
- "learning_rate": 1.5403225806451613e-06,
1172
- "loss": 3.7483,
1173
- "mean_token_accuracy": 0.9196753472089767,
1174
- "num_tokens": 27086411.0,
1175
- "step": 1060,
1176
- "teacher_ce_loss": 2.028025
1177
- },
1178
- {
1179
- "alignment_loss": 0.526774,
1180
- "epoch": 1.712,
1181
- "grad_norm": 39.715152740478516,
1182
- "learning_rate": 1.4596774193548387e-06,
1183
- "loss": 3.6202,
1184
- "mean_token_accuracy": 0.9165695428848266,
1185
- "num_tokens": 27399691.0,
1186
- "step": 1070,
1187
- "teacher_ce_loss": 1.48792
1188
- },
1189
- {
1190
- "alignment_loss": 0.551694,
1191
- "epoch": 1.728,
1192
- "grad_norm": 45.96261215209961,
1193
- "learning_rate": 1.3790322580645163e-06,
1194
- "loss": 3.9325,
1195
- "mean_token_accuracy": 0.9174860268831253,
1196
- "num_tokens": 27714499.0,
1197
- "step": 1080,
1198
- "teacher_ce_loss": 2.094843
1199
- },
1200
- {
1201
- "alignment_loss": 0.516514,
1202
- "epoch": 1.744,
1203
- "grad_norm": 36.325862884521484,
1204
- "learning_rate": 1.2983870967741937e-06,
1205
- "loss": 3.7197,
1206
- "mean_token_accuracy": 0.9220000863075256,
1207
- "num_tokens": 28028539.0,
1208
- "step": 1090,
1209
- "teacher_ce_loss": 1.576362
1210
- },
1211
- {
1212
- "alignment_loss": 0.532741,
1213
- "epoch": 1.76,
1214
- "grad_norm": 41.12571334838867,
1215
- "learning_rate": 1.2177419354838711e-06,
1216
- "loss": 3.2402,
1217
- "mean_token_accuracy": 0.9316521614789963,
1218
- "num_tokens": 28343606.0,
1219
- "step": 1100,
1220
- "teacher_ce_loss": 1.504065
1221
- },
1222
- {
1223
- "alignment_loss": 0.516031,
1224
- "epoch": 1.776,
1225
- "grad_norm": 40.023616790771484,
1226
- "learning_rate": 1.1370967741935485e-06,
1227
- "loss": 3.4433,
1228
- "mean_token_accuracy": 0.9273842662572861,
1229
- "num_tokens": 28657957.0,
1230
- "step": 1110,
1231
- "teacher_ce_loss": 1.73723
1232
- },
1233
- {
1234
- "alignment_loss": 0.491594,
1235
- "epoch": 1.792,
1236
- "grad_norm": 46.13947296142578,
1237
- "learning_rate": 1.056451612903226e-06,
1238
- "loss": 3.7568,
1239
- "mean_token_accuracy": 0.9203128784894943,
1240
- "num_tokens": 28975028.0,
1241
- "step": 1120,
1242
- "teacher_ce_loss": 2.01769
1243
- },
1244
- {
1245
- "alignment_loss": 0.536311,
1246
- "epoch": 1.808,
1247
- "grad_norm": 42.01000213623047,
1248
- "learning_rate": 9.758064516129033e-07,
1249
- "loss": 3.9123,
1250
- "mean_token_accuracy": 0.917031443119049,
1251
- "num_tokens": 29291150.0,
1252
- "step": 1130,
1253
- "teacher_ce_loss": 1.913613
1254
- },
1255
- {
1256
- "alignment_loss": 0.524734,
1257
- "epoch": 1.8239999999999998,
1258
- "grad_norm": 45.14901351928711,
1259
- "learning_rate": 8.951612903225807e-07,
1260
- "loss": 4.1744,
1261
- "mean_token_accuracy": 0.9157502561807632,
1262
- "num_tokens": 29610786.0,
1263
- "step": 1140,
1264
- "teacher_ce_loss": 2.204652
1265
- },
1266
- {
1267
- "alignment_loss": 0.524073,
1268
- "epoch": 1.8399999999999999,
1269
- "grad_norm": 40.34686279296875,
1270
- "learning_rate": 8.145161290322581e-07,
1271
- "loss": 3.1054,
1272
- "mean_token_accuracy": 0.93112713098526,
1273
- "num_tokens": 29925770.0,
1274
- "step": 1150,
1275
- "teacher_ce_loss": 1.589528
1276
- },
1277
- {
1278
- "alignment_loss": 0.52039,
1279
- "epoch": 1.8559999999999999,
1280
- "grad_norm": 45.0685920715332,
1281
- "learning_rate": 7.338709677419354e-07,
1282
- "loss": 3.7313,
1283
- "mean_token_accuracy": 0.9227394700050354,
1284
- "num_tokens": 30241771.0,
1285
- "step": 1160,
1286
- "teacher_ce_loss": 2.025996
1287
- },
1288
- {
1289
- "alignment_loss": 0.543487,
1290
- "epoch": 1.8719999999999999,
1291
- "grad_norm": 39.49046325683594,
1292
- "learning_rate": 6.532258064516129e-07,
1293
- "loss": 3.5147,
1294
- "mean_token_accuracy": 0.9247892439365387,
1295
- "num_tokens": 30553503.0,
1296
- "step": 1170,
1297
- "teacher_ce_loss": 2.050541
1298
- },
1299
- {
1300
- "alignment_loss": 0.494009,
1301
- "epoch": 1.888,
1302
- "grad_norm": 35.7860107421875,
1303
- "learning_rate": 5.725806451612903e-07,
1304
- "loss": 3.583,
1305
- "mean_token_accuracy": 0.9248492360115051,
1306
- "num_tokens": 30868927.0,
1307
- "step": 1180,
1308
- "teacher_ce_loss": 1.920703
1309
- },
1310
- {
1311
- "alignment_loss": 0.519892,
1312
- "epoch": 1.904,
1313
- "grad_norm": 45.415245056152344,
1314
- "learning_rate": 4.919354838709677e-07,
1315
- "loss": 3.4943,
1316
- "mean_token_accuracy": 0.9268854647874832,
1317
- "num_tokens": 31182975.0,
1318
- "step": 1190,
1319
- "teacher_ce_loss": 1.84003
1320
- },
1321
- {
1322
- "alignment_loss": 0.520323,
1323
- "epoch": 1.92,
1324
- "grad_norm": 40.65234375,
1325
- "learning_rate": 4.112903225806452e-07,
1326
- "loss": 3.6577,
1327
- "mean_token_accuracy": 0.921201354265213,
1328
- "num_tokens": 31499558.0,
1329
- "step": 1200,
1330
- "teacher_ce_loss": 2.024774
1331
- }
1332
- ],
1333
- "logging_steps": 10,
1334
- "max_steps": 1250,
1335
- "num_input_tokens_seen": 0,
1336
- "num_train_epochs": 2,
1337
- "save_steps": 100,
1338
- "stateful_callbacks": {
1339
- "TrainerControl": {
1340
- "args": {
1341
- "should_epoch_stop": false,
1342
- "should_evaluate": false,
1343
- "should_log": false,
1344
- "should_save": true,
1345
- "should_training_stop": false
1346
- },
1347
- "attributes": {}
1348
- }
1349
- },
1350
- "total_flos": 3.6381618424732713e+18,
1351
- "train_batch_size": 8,
1352
- "trial_name": null,
1353
- "trial_params": null
1354
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6177d8ea45173ed89af768c4e6df4644fb1d5798032b15f939841a224970f390
3
- size 11107
 
 
 
 
size-80k/checkpoint-1200/video_preprocessor_config.json DELETED
@@ -1,43 +0,0 @@
1
- {
2
- "crop_size": null,
3
- "data_format": "channels_first",
4
- "default_to_square": true,
5
- "device": null,
6
- "do_center_crop": null,
7
- "do_convert_rgb": true,
8
- "do_normalize": true,
9
- "do_pad": null,
10
- "do_rescale": true,
11
- "do_resize": true,
12
- "do_sample_frames": false,
13
- "fps": null,
14
- "image_mean": [
15
- 0.48145466,
16
- 0.4578275,
17
- 0.40821073
18
- ],
19
- "image_std": [
20
- 0.26862954,
21
- 0.26130258,
22
- 0.27577711
23
- ],
24
- "input_data_format": null,
25
- "max_frames": 768,
26
- "max_pixels": 12845056,
27
- "merge_size": 2,
28
- "min_frames": 4,
29
- "min_pixels": 3136,
30
- "num_frames": null,
31
- "patch_size": 14,
32
- "processor_class": "Qwen2_5_VLProcessor",
33
- "resample": 3,
34
- "rescale_factor": 0.00392156862745098,
35
- "size": {
36
- "longest_edge": 12845056,
37
- "shortest_edge": 3136
38
- },
39
- "size_divisor": null,
40
- "temporal_patch_size": 2,
41
- "video_metadata": null,
42
- "video_processor_type": "Qwen2VLVideoProcessor"
43
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
size-80k/checkpoint-1200/vocab.json DELETED
The diff for this file is too large to render. See raw diff
 
size-80k/checkpoint-1200/zero_to_fp32.py DELETED
@@ -1,760 +0,0 @@
1
- #!/usr/bin/env python
2
-
3
- # Copyright (c) Microsoft Corporation.
4
- # SPDX-License-Identifier: Apache-2.0
5
-
6
- # DeepSpeed Team
7
-
8
- # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
- # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
- # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
- # application.
12
- #
13
- # example:
14
- # python zero_to_fp32.py . output_dir/
15
- # or
16
- # python zero_to_fp32.py . output_dir/ --safe_serialization
17
-
18
- import argparse
19
- import torch
20
- import glob
21
- import math
22
- import os
23
- import re
24
- import gc
25
- import json
26
- import numpy as np
27
- from tqdm import tqdm
28
- from collections import OrderedDict
29
- from dataclasses import dataclass
30
-
31
- # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
- # DeepSpeed data structures it has to be available in the current python environment.
33
- from deepspeed.utils import logger
34
- from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
- FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
- FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
-
38
-
39
- @dataclass
40
- class zero_model_state:
41
- buffers: dict()
42
- param_shapes: dict()
43
- shared_params: list
44
- ds_version: int
45
- frozen_param_shapes: dict()
46
- frozen_param_fragments: dict()
47
-
48
-
49
- debug = 0
50
-
51
- # load to cpu
52
- device = torch.device('cpu')
53
-
54
-
55
- def atoi(text):
56
- return int(text) if text.isdigit() else text
57
-
58
-
59
- def natural_keys(text):
60
- '''
61
- alist.sort(key=natural_keys) sorts in human order
62
- http://nedbatchelder.com/blog/200712/human_sorting.html
63
- (See Toothy's implementation in the comments)
64
- '''
65
- return [atoi(c) for c in re.split(r'(\d+)', text)]
66
-
67
-
68
- def get_model_state_file(checkpoint_dir, zero_stage):
69
- if not os.path.isdir(checkpoint_dir):
70
- raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
-
72
- # there should be only one file
73
- if zero_stage <= 2:
74
- file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
- elif zero_stage == 3:
76
- file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
-
78
- if not os.path.exists(file):
79
- raise FileNotFoundError(f"can't find model states file at '{file}'")
80
-
81
- return file
82
-
83
-
84
- def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
- # XXX: need to test that this simple glob rule works for multi-node setup too
86
- ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
-
88
- if len(ckpt_files) == 0:
89
- raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
-
91
- return ckpt_files
92
-
93
-
94
- def get_optim_files(checkpoint_dir):
95
- return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
-
97
-
98
- def get_model_state_files(checkpoint_dir):
99
- return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
-
101
-
102
- def parse_model_states(files):
103
- zero_model_states = []
104
- for file in files:
105
- state_dict = torch.load(file, map_location=device, weights_only=False)
106
-
107
- if BUFFER_NAMES not in state_dict:
108
- raise ValueError(f"{file} is not a model state checkpoint")
109
- buffer_names = state_dict[BUFFER_NAMES]
110
- if debug:
111
- print("Found buffers:", buffer_names)
112
-
113
- # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
- buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
- param_shapes = state_dict[PARAM_SHAPES]
116
-
117
- # collect parameters that are included in param_shapes
118
- param_names = []
119
- for s in param_shapes:
120
- for name in s.keys():
121
- param_names.append(name)
122
-
123
- # update with frozen parameters
124
- frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
- if frozen_param_shapes is not None:
126
- if debug:
127
- print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
- param_names += list(frozen_param_shapes.keys())
129
-
130
- # handle shared params
131
- shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
-
133
- ds_version = state_dict.get(DS_VERSION, None)
134
-
135
- frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
-
137
- z_model_state = zero_model_state(buffers=buffers,
138
- param_shapes=param_shapes,
139
- shared_params=shared_params,
140
- ds_version=ds_version,
141
- frozen_param_shapes=frozen_param_shapes,
142
- frozen_param_fragments=frozen_param_fragments)
143
- zero_model_states.append(z_model_state)
144
-
145
- return zero_model_states
146
-
147
-
148
- def parse_optim_states(files, ds_checkpoint_dir):
149
- total_files = len(files)
150
- state_dicts = []
151
- for f in tqdm(files, desc='Loading checkpoint shards'):
152
- state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
- # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
- # and also handle the case where it was already removed by another helper script
155
- state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
- state_dicts.append(state_dict)
157
-
158
- if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
- raise ValueError(f"{files[0]} is not a zero checkpoint")
160
- zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
- world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
-
163
- # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
- # parameters can be different from data parallelism for non-expert parameters. So we can just
165
- # use the max of the partition_count to get the dp world_size.
166
-
167
- if type(world_size) is list:
168
- world_size = max(world_size)
169
-
170
- if world_size != total_files:
171
- raise ValueError(
172
- f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
- "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
- )
175
-
176
- # the groups are named differently in each stage
177
- if zero_stage <= 2:
178
- fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
- elif zero_stage == 3:
180
- fp32_groups_key = FP32_FLAT_GROUPS
181
- else:
182
- raise ValueError(f"unknown zero stage {zero_stage}")
183
-
184
- fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
- return zero_stage, world_size, fp32_flat_groups
186
-
187
-
188
- def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
- """
190
- Returns fp32 state_dict reconstructed from ds checkpoint
191
-
192
- Args:
193
- - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
-
195
- """
196
- print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
-
198
- optim_files = get_optim_files(ds_checkpoint_dir)
199
- zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
- print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
-
202
- model_files = get_model_state_files(ds_checkpoint_dir)
203
-
204
- zero_model_states = parse_model_states(model_files)
205
- print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
-
207
- if zero_stage <= 2:
208
- return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
- exclude_frozen_parameters)
210
- elif zero_stage == 3:
211
- return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
- exclude_frozen_parameters)
213
-
214
-
215
- def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
- if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
- return
218
-
219
- frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
- frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
-
222
- if debug:
223
- num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
- print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
-
226
- wanted_params = len(frozen_param_shapes)
227
- wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
- avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
- print(f'Frozen params: Have {avail_numel} numels to process.')
230
- print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
-
232
- total_params = 0
233
- total_numel = 0
234
- for name, shape in frozen_param_shapes.items():
235
- total_params += 1
236
- unpartitioned_numel = shape.numel()
237
- total_numel += unpartitioned_numel
238
-
239
- state_dict[name] = frozen_param_fragments[name]
240
-
241
- if debug:
242
- print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
-
244
- print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
-
246
-
247
- def _has_callable(obj, fn):
248
- attr = getattr(obj, fn, None)
249
- return callable(attr)
250
-
251
-
252
- def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
- param_shapes = zero_model_states[0].param_shapes
254
-
255
- # Reconstruction protocol:
256
- #
257
- # XXX: document this
258
-
259
- if debug:
260
- for i in range(world_size):
261
- for j in range(len(fp32_flat_groups[0])):
262
- print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
-
264
- # XXX: memory usage doubles here (zero2)
265
- num_param_groups = len(fp32_flat_groups[0])
266
- merged_single_partition_of_fp32_groups = []
267
- for i in range(num_param_groups):
268
- merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
- full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
- merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
- avail_numel = sum(
272
- [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
-
274
- if debug:
275
- wanted_params = sum([len(shapes) for shapes in param_shapes])
276
- wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
- # not asserting if there is a mismatch due to possible padding
278
- print(f"Have {avail_numel} numels to process.")
279
- print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
-
281
- # params
282
- # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
- # out-of-core computing solution
284
- total_numel = 0
285
- total_params = 0
286
- for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
- offset = 0
288
- avail_numel = full_single_fp32_vector.numel()
289
- for name, shape in shapes.items():
290
-
291
- unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
- total_numel += unpartitioned_numel
293
- total_params += 1
294
-
295
- if debug:
296
- print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
- state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
- offset += unpartitioned_numel
299
-
300
- # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
- # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
- # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
- # live optimizer object, so we are checking that the numbers are within the right range
304
- align_to = 2 * world_size
305
-
306
- def zero2_align(x):
307
- return align_to * math.ceil(x / align_to)
308
-
309
- if debug:
310
- print(f"original offset={offset}, avail_numel={avail_numel}")
311
-
312
- offset = zero2_align(offset)
313
- avail_numel = zero2_align(avail_numel)
314
-
315
- if debug:
316
- print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
-
318
- # Sanity check
319
- if offset != avail_numel:
320
- raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
-
322
- print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
-
324
-
325
- def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
- exclude_frozen_parameters):
327
- state_dict = OrderedDict()
328
-
329
- # buffers
330
- buffers = zero_model_states[0].buffers
331
- state_dict.update(buffers)
332
- if debug:
333
- print(f"added {len(buffers)} buffers")
334
-
335
- if not exclude_frozen_parameters:
336
- _zero2_merge_frozen_params(state_dict, zero_model_states)
337
-
338
- _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
-
340
- # recover shared parameters
341
- for pair in zero_model_states[0].shared_params:
342
- if pair[1] in state_dict:
343
- state_dict[pair[0]] = state_dict[pair[1]]
344
-
345
- return state_dict
346
-
347
-
348
- def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
- remainder = unpartitioned_numel % world_size
350
- padding_numel = (world_size - remainder) if remainder else 0
351
- partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
- return partitioned_numel, padding_numel
353
-
354
-
355
- def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
- if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
- return
358
-
359
- if debug:
360
- for i in range(world_size):
361
- num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
- print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
-
364
- frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
- wanted_params = len(frozen_param_shapes)
366
- wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
- avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
- print(f'Frozen params: Have {avail_numel} numels to process.')
369
- print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
-
371
- total_params = 0
372
- total_numel = 0
373
- for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
- total_params += 1
375
- unpartitioned_numel = shape.numel()
376
- total_numel += unpartitioned_numel
377
-
378
- param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
- state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
-
381
- partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
-
383
- if debug:
384
- print(
385
- f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
- )
387
-
388
- print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
-
390
-
391
- class GatheredTensor:
392
- """
393
- A pseudo tensor that collects partitioned weights.
394
- It is more memory efficient when there are multiple groups.
395
- """
396
-
397
- def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
- self.flat_groups = flat_groups
399
- self.flat_groups_offset = flat_groups_offset
400
- self.offset = offset
401
- self.partitioned_numel = partitioned_numel
402
- self.shape = shape
403
- self.dtype = self.flat_groups[0][0].dtype
404
-
405
- def contiguous(self):
406
- """
407
- Merge partitioned weights from flat_groups into a single tensor.
408
- """
409
- end_idx = self.offset + self.partitioned_numel
410
- world_size = len(self.flat_groups)
411
- pad_flat_param_chunks = []
412
-
413
- for rank_i in range(world_size):
414
- # for each rank, we need to collect weights from related group/groups
415
- flat_groups_at_rank_i = self.flat_groups[rank_i]
416
- start_group_id = None
417
- end_group_id = None
418
- for group_id in range(len(self.flat_groups_offset)):
419
- if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
- start_group_id = group_id
421
- if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
- end_group_id = group_id
423
- break
424
- # collect weights from related group/groups
425
- for group_id in range(start_group_id, end_group_id + 1):
426
- flat_tensor = flat_groups_at_rank_i[group_id]
427
- start_offset = self.offset - self.flat_groups_offset[group_id]
428
- end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
- pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
-
431
- # collect weights from all ranks
432
- pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
- param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
- return param
435
-
436
-
437
- def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
- param_shapes = zero_model_states[0].param_shapes
439
- avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
-
441
- # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
- # param, re-consolidating each param, while dealing with padding if any
443
-
444
- # merge list of dicts, preserving order
445
- param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
-
447
- if debug:
448
- for i in range(world_size):
449
- print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
-
451
- wanted_params = len(param_shapes)
452
- wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
- # not asserting if there is a mismatch due to possible padding
454
- avail_numel = fp32_flat_groups[0].numel() * world_size
455
- print(f"Trainable params: Have {avail_numel} numels to process.")
456
- print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
-
458
- # params
459
- # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
- # out-of-core computing solution
461
- offset = 0
462
- total_numel = 0
463
- total_params = 0
464
- flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
- for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
- unpartitioned_numel = shape.numel()
467
- total_numel += unpartitioned_numel
468
- total_params += 1
469
- partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
-
471
- if debug:
472
- print(
473
- f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
- )
475
-
476
- # memory efficient tensor
477
- tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
- state_dict[name] = tensor
479
- offset += partitioned_numel
480
-
481
- offset *= world_size
482
-
483
- # Sanity check
484
- if offset != avail_numel:
485
- raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
-
487
- print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
-
489
-
490
- def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
- exclude_frozen_parameters):
492
- state_dict = OrderedDict()
493
-
494
- # buffers
495
- buffers = zero_model_states[0].buffers
496
- state_dict.update(buffers)
497
- if debug:
498
- print(f"added {len(buffers)} buffers")
499
-
500
- if not exclude_frozen_parameters:
501
- _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
-
503
- _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
-
505
- # recover shared parameters
506
- for pair in zero_model_states[0].shared_params:
507
- if pair[1] in state_dict:
508
- state_dict[pair[0]] = state_dict[pair[1]]
509
-
510
- return state_dict
511
-
512
-
513
- def to_torch_tensor(state_dict, return_empty_tensor=False):
514
- """
515
- Convert state_dict of GatheredTensor to torch tensor
516
- """
517
- torch_state_dict = {}
518
- converted_tensors = {}
519
- for name, tensor in state_dict.items():
520
- tensor_id = id(tensor)
521
- if tensor_id in converted_tensors: # shared tensors
522
- shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
- torch_state_dict[name] = shared_tensor
524
- else:
525
- converted_tensors[tensor_id] = name
526
- if return_empty_tensor:
527
- torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
- else:
529
- torch_state_dict[name] = tensor.contiguous()
530
- return torch_state_dict
531
-
532
-
533
- def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
- tag=None,
535
- exclude_frozen_parameters=False,
536
- lazy_mode=False):
537
- """
538
- Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
- ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
- via a model hub.
541
-
542
- Args:
543
- - ``checkpoint_dir``: path to the desired checkpoint folder
544
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
- - ``exclude_frozen_parameters``: exclude frozen parameters
546
- - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
- Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
-
549
- Returns:
550
- - pytorch ``state_dict``
551
-
552
- A typical usage might be ::
553
-
554
- from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
- # do the training and checkpoint saving
556
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
- model = model.cpu() # move to cpu
558
- model.load_state_dict(state_dict)
559
- # submit to model hub or save the model to share with others
560
-
561
- In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
- application. i.e. you will need to re-initialize the deepspeed engine, since
563
- ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
-
565
- If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
-
567
- Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
- You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
- the checkpoint. Or you can load state_dict in lazy mode ::
570
-
571
- from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
- for name, lazy_tensor in state_dict.item():
574
- tensor = lazy_tensor.contiguous() # to cpu
575
- print(name, tensor)
576
- # del tensor to release memory if it no longer in use
577
- """
578
- if tag is None:
579
- latest_path = os.path.join(checkpoint_dir, 'latest')
580
- if os.path.isfile(latest_path):
581
- with open(latest_path, 'r') as fd:
582
- tag = fd.read().strip()
583
- else:
584
- raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
-
586
- ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
-
588
- if not os.path.isdir(ds_checkpoint_dir):
589
- raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
-
591
- state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
- if lazy_mode:
593
- return state_dict
594
- else:
595
- return to_torch_tensor(state_dict)
596
-
597
-
598
- def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
- output_dir,
600
- max_shard_size="5GB",
601
- safe_serialization=False,
602
- tag=None,
603
- exclude_frozen_parameters=False):
604
- """
605
- Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
- loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
-
608
- Args:
609
- - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
- - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
- - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
- - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
- - ``exclude_frozen_parameters``: exclude frozen parameters
615
- """
616
-
617
- # Dependency pre-check
618
- if safe_serialization:
619
- try:
620
- from safetensors.torch import save_file
621
- except ImportError:
622
- print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
- raise
624
- if max_shard_size is not None:
625
- try:
626
- from huggingface_hub import split_torch_state_dict_into_shards
627
- except ImportError:
628
- print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
- raise
630
-
631
- # Convert zero checkpoint to state_dict
632
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
- tag,
634
- exclude_frozen_parameters,
635
- lazy_mode=True)
636
-
637
- # Shard the model if it is too big.
638
- weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
- if max_shard_size is not None:
640
- filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
- # an memory-efficient approach for sharding
642
- empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
- state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
- filename_pattern=filename_pattern,
645
- max_shard_size=max_shard_size)
646
- else:
647
- from collections import namedtuple
648
- StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
- state_dict_split = StateDictSplit(is_sharded=False,
650
- filename_to_tensors={weights_name: list(state_dict.keys())})
651
-
652
- # Save the model by shard
653
- os.makedirs(output_dir, exist_ok=True)
654
- filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
- for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
- shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
- shard_state_dict = to_torch_tensor(shard_state_dict)
658
- output_path = os.path.join(output_dir, shard_file)
659
- if safe_serialization:
660
- save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
- else:
662
- torch.save(shard_state_dict, output_path)
663
- # release the memory of current shard
664
- for tensor_name in list(shard_state_dict.keys()):
665
- del state_dict[tensor_name]
666
- del shard_state_dict[tensor_name]
667
- del shard_state_dict
668
- gc.collect()
669
-
670
- # Save index if sharded
671
- if state_dict_split.is_sharded:
672
- index = {
673
- "metadata": state_dict_split.metadata,
674
- "weight_map": state_dict_split.tensor_to_filename,
675
- }
676
- save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
- save_index_file = os.path.join(output_dir, save_index_file)
678
- with open(save_index_file, "w", encoding="utf-8") as f:
679
- content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
- f.write(content)
681
-
682
-
683
- def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
- """
685
- 1. Put the provided model to cpu
686
- 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
- 3. Load it into the provided model
688
-
689
- Args:
690
- - ``model``: the model object to update
691
- - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
- - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
-
694
- Returns:
695
- - ``model`: modified model
696
-
697
- Make sure you have plenty of CPU memory available before you call this function. If you don't
698
- have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
- conveniently placed for you in the checkpoint folder.
700
-
701
- A typical usage might be ::
702
-
703
- from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
- model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
- # submit to model hub or save the model to share with others
706
-
707
- Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
- of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
- ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
-
711
- """
712
- logger.info(f"Extracting fp32 weights")
713
- state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
-
715
- logger.info(f"Overwriting model with fp32 weights")
716
- model = model.cpu()
717
- model.load_state_dict(state_dict, strict=False)
718
-
719
- return model
720
-
721
-
722
- if __name__ == "__main__":
723
- parser = argparse.ArgumentParser()
724
- parser.add_argument("checkpoint_dir",
725
- type=str,
726
- help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
- parser.add_argument("output_dir",
728
- type=str,
729
- help="directory to the pytorch fp32 state_dict output files"
730
- "(e.g. path/checkpoint-12-output/)")
731
- parser.add_argument(
732
- "--max_shard_size",
733
- type=str,
734
- default="5GB",
735
- help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
- "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
- "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
- "without CPU OOM issues.")
739
- parser.add_argument(
740
- "--safe_serialization",
741
- default=False,
742
- action='store_true',
743
- help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
- parser.add_argument("-t",
745
- "--tag",
746
- type=str,
747
- default=None,
748
- help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
- parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
- parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
- args = parser.parse_args()
752
-
753
- debug = args.debug
754
-
755
- convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
- args.output_dir,
757
- max_shard_size=args.max_shard_size,
758
- safe_serialization=args.safe_serialization,
759
- tag=args.tag,
760
- exclude_frozen_parameters=args.exclude_frozen_parameters)