Hugging Face's logo

happyhappy-jun
/
abcd2a_control_ft_step6000

Safetensors
Gr00tN1d6
Model card Files
xet
 Community
happyhappy-jun commited on
Commit
ba063d9
·
verified ·
1 Parent(s): ad7fc86

Upload folder using huggingface_hub

Browse files
Files changed (35) hide show
  1. config.json +70 -0
  2. embodiment_id.json +11 -0
  3. experiment_cfg/conf.yaml +228 -0
  4. experiment_cfg/config.yaml +258 -0
  5. experiment_cfg/dataset_statistics.json +317 -0
  6. experiment_cfg/final_model_config.json +53 -0
  7. experiment_cfg/final_processor_config.json +0 -0
  8. global_step6000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +3 -0
  9. global_step6000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +3 -0
  10. global_step6000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt +3 -0
  11. global_step6000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt +3 -0
  12. global_step6000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt +3 -0
  13. global_step6000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt +3 -0
  14. global_step6000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt +3 -0
  15. global_step6000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt +3 -0
  16. global_step6000/mp_rank_00_model_states.pt +3 -0
  17. latest +1 -0
  18. model-00001-of-00002.safetensors +3 -0
  19. model-00002-of-00002.safetensors +3 -0
  20. model.safetensors.index.json +0 -0
  21. processor_config.json +480 -0
  22. rng_state_0.pth +3 -0
  23. rng_state_1.pth +3 -0
  24. rng_state_2.pth +3 -0
  25. rng_state_3.pth +3 -0
  26. rng_state_4.pth +3 -0
  27. rng_state_5.pth +3 -0
  28. rng_state_6.pth +3 -0
  29. rng_state_7.pth +3 -0
  30. scheduler.pt +3 -0
  31. statistics.json +0 -0
  32. trainer_state.json +3634 -0
  33. training_args.bin +3 -0
  34. wandb_config.json +1 -0
  35. zero_to_fp32.py +760 -0
config.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_horizon": 50,
3
+ "add_pos_embed": true,
4
+ "apply_sincos_state_encoding": true,
5
+ "architectures": [
6
+ "Gr00tN1d6"
7
+ ],
8
+ "attn_dropout": 0.2,
9
+ "attn_implementation": null,
10
+ "backbone_embedding_dim": 2048,
11
+ "backbone_model_type": "eagle",
12
+ "backbone_trainable_params_fp32": true,
13
+ "collator_overwrite_image_inputs": false,
14
+ "color_jitter_params": {
15
+ "brightness": 0.1,
16
+ "contrast": 0.1,
17
+ "hue": 0.1,
18
+ "saturation": 0.1
19
+ },
20
+ "crop_fraction": 0.95,
21
+ "diffusion_model_cfg": {
22
+ "attention_head_dim": 48,
23
+ "dropout": 0.2,
24
+ "final_dropout": true,
25
+ "interleave_self_attention": true,
26
+ "norm_type": "ada_norm",
27
+ "num_attention_heads": 32,
28
+ "num_layers": 32,
29
+ "output_dim": 1024,
30
+ "positional_embeddings": null
31
+ },
32
+ "eagle_collator": true,
33
+ "formalize_language": true,
34
+ "gemma_collator": false,
35
+ "hidden_size": 1024,
36
+ "image_crop_size": null,
37
+ "image_target_size": null,
38
+ "input_embedding_dim": 1536,
39
+ "load_bf16": true,
40
+ "max_action_dim": 128,
41
+ "max_num_embodiments": 32,
42
+ "max_seq_len": 1024,
43
+ "max_state_dim": 128,
44
+ "model_dtype": "bfloat16",
45
+ "model_name": "nvidia/Eagle-Block2A-2B-v2",
46
+ "model_type": "Gr00tN1d6",
47
+ "noise_beta_alpha": 1.5,
48
+ "noise_beta_beta": 1.0,
49
+ "noise_s": 0.999,
50
+ "num_inference_timesteps": 4,
51
+ "num_timestep_buckets": 1000,
52
+ "random_rotation_angle": null,
53
+ "reproject_vision": false,
54
+ "select_layer": 16,
55
+ "shortest_image_edge": 256,
56
+ "state_dropout_prob": 0.0,
57
+ "torch_dtype": "bfloat16",
58
+ "transformers_version": "4.51.3",
59
+ "tune_diffusion_model": true,
60
+ "tune_llm": false,
61
+ "tune_projector": true,
62
+ "tune_top_llm_layers": 4,
63
+ "tune_visual": false,
64
+ "tune_vlln": true,
65
+ "use_albumentations_transforms": true,
66
+ "use_alternate_vl_dit": true,
67
+ "use_flash_attention": true,
68
+ "use_relative_action": true,
69
+ "use_vlln": true
70
+ }
embodiment_id.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "robocasa_panda_omron": 13,
3
+ "gr1": 20,
4
+ "behavior_r1_pro": 24,
5
+ "unitree_g1": 8,
6
+ "oxe_google": 0,
7
+ "oxe_widowx": 1,
8
+ "libero_panda": 2,
9
+ "oxe_droid": 16,
10
+ "new_embodiment": 10
11
+ }
experiment_cfg/conf.yaml ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ load_config_path: null
2
+ model:
3
+ model_type: Gr00tN1d6
4
+ model_dtype: bfloat16
5
+ model_name: nvidia/Eagle-Block2A-2B-v2
6
+ backbone_model_type: eagle
7
+ model_revision: null
8
+ tune_top_llm_layers: 4
9
+ backbone_embedding_dim: 2048
10
+ tune_llm: false
11
+ tune_visual: false
12
+ select_layer: 16
13
+ reproject_vision: false
14
+ use_flash_attention: true
15
+ load_bf16: true
16
+ collator_overwrite_image_inputs: false
17
+ eagle_collator: true
18
+ backbone_trainable_params_fp32: true
19
+ image_crop_size: null
20
+ image_target_size: null
21
+ shortest_image_edge: 256
22
+ crop_fraction: 0.95
23
+ random_rotation_angle: null
24
+ color_jitter_params: null
25
+ use_albumentations_transforms: true
26
+ formalize_language: true
27
+ apply_sincos_state_encoding: false
28
+ use_relative_action: true
29
+ max_state_dim: 29
30
+ max_action_dim: 29
31
+ action_horizon: 16
32
+ hidden_size: 1024
33
+ input_embedding_dim: 1536
34
+ add_pos_embed: true
35
+ attn_dropout: 0.2
36
+ use_vlln: true
37
+ max_seq_len: 1024
38
+ use_alternate_vl_dit: true
39
+ attend_text_every_n_blocks: 2
40
+ diffusion_model_cfg:
41
+ positional_embeddings: null
42
+ num_layers: 32
43
+ num_attention_heads: 32
44
+ attention_head_dim: 48
45
+ norm_type: ada_norm
46
+ dropout: 0.2
47
+ final_dropout: true
48
+ output_dim: 1024
49
+ interleave_self_attention: true
50
+ num_inference_timesteps: 4
51
+ noise_beta_alpha: 1.5
52
+ noise_beta_beta: 1.0
53
+ noise_s: 0.999
54
+ num_timestep_buckets: 1000
55
+ tune_projector: true
56
+ tune_diffusion_model: true
57
+ tune_vlln: true
58
+ state_dropout_prob: 0.0
59
+ state_additive_noise_scale: 0.0
60
+ max_num_embodiments: 32
61
+ data:
62
+ datasets:
63
+ - dataset_paths:
64
+ - /fsx/ubuntu/atomicvla/data/robocasa365_subset_atomic_x2/v1.0/pretrain/atomic/CloseBlenderLid/20250822/lerobot
65
+ embodiment_tag: new_embodiment
66
+ mix_ratio: 1.0
67
+ dataset_type: physical_embodiment
68
+ val_dataset_path: null
69
+ modality_configs:
70
+ new_embodiment:
71
+ video:
72
+ delta_indices:
73
+ - 0
74
+ modality_keys:
75
+ - robot0_eye_in_hand
76
+ - robot0_agentview_left
77
+ - robot0_agentview_right
78
+ sin_cos_embedding_keys: null
79
+ mean_std_embedding_keys: null
80
+ action_configs: null
81
+ state:
82
+ delta_indices:
83
+ - 0
84
+ modality_keys:
85
+ - base_position
86
+ - base_rotation
87
+ - end_effector_position_relative
88
+ - end_effector_rotation_relative
89
+ - gripper_qpos
90
+ sin_cos_embedding_keys: null
91
+ mean_std_embedding_keys: null
92
+ action_configs: null
93
+ action:
94
+ delta_indices:
95
+ - 0
96
+ - 1
97
+ - 2
98
+ - 3
99
+ - 4
100
+ - 5
101
+ - 6
102
+ - 7
103
+ - 8
104
+ - 9
105
+ - 10
106
+ - 11
107
+ - 12
108
+ - 13
109
+ - 14
110
+ - 15
111
+ modality_keys:
112
+ - base_motion
113
+ - control_mode
114
+ - end_effector_position
115
+ - end_effector_rotation
116
+ - gripper_close
117
+ sin_cos_embedding_keys: null
118
+ mean_std_embedding_keys: null
119
+ action_configs:
120
+ - rep: ABSOLUTE
121
+ type: NON_EEF
122
+ format: DEFAULT
123
+ state_key: null
124
+ - rep: ABSOLUTE
125
+ type: NON_EEF
126
+ format: DEFAULT
127
+ state_key: null
128
+ - rep: ABSOLUTE
129
+ type: EEF
130
+ format: DEFAULT
131
+ state_key: null
132
+ - rep: ABSOLUTE
133
+ type: EEF
134
+ format: DEFAULT
135
+ state_key: null
136
+ - rep: ABSOLUTE
137
+ type: NON_EEF
138
+ format: DEFAULT
139
+ state_key: null
140
+ language:
141
+ delta_indices:
142
+ - 0
143
+ modality_keys:
144
+ - sub_task
145
+ sin_cos_embedding_keys: null
146
+ mean_std_embedding_keys: null
147
+ action_configs: null
148
+ download_cache: false
149
+ shard_size: 1024
150
+ episode_sampling_rate: 0.1
151
+ num_shards_per_epoch: 100000
152
+ override_pretraining_statistics: false
153
+ mode: single_turn
154
+ random_chop: 0.0
155
+ mock_dataset_mode: false
156
+ shuffle: true
157
+ seed: 42
158
+ multiprocessing_context: fork
159
+ allow_padding: false
160
+ subsample_ratio: 1.0
161
+ image_crop_size:
162
+ - 244
163
+ - 244
164
+ image_target_size:
165
+ - 224
166
+ - 224
167
+ video_backend: torchcodec
168
+ training:
169
+ output_dir: ./ckpt/abcd2a_control_ft
170
+ experiment_name: null
171
+ max_steps: 10000
172
+ global_batch_size: 128
173
+ batch_size: null
174
+ gradient_accumulation_steps: 1
175
+ learning_rate: 0.0001
176
+ lr_scheduler_type: cosine
177
+ weight_decay: 1.0e-05
178
+ warmup_ratio: 0.05
179
+ warmup_steps: 0
180
+ max_grad_norm: 1.0
181
+ wsd_decay_steps: 3000
182
+ wsd_min_lr_ratio: 0.0
183
+ wsd_decay_type: cosine
184
+ reset_optimizer: false
185
+ optim: adamw_torch
186
+ start_from_checkpoint: ./ckpt/abcd2a_control_pt
187
+ tf32: true
188
+ fp16: false
189
+ bf16: true
190
+ eval_bf16: true
191
+ logging_steps: 10
192
+ save_steps: 2000
193
+ save_total_limit: 5
194
+ save_vl_model: false
195
+ upload_checkpoints: false
196
+ upload_every: 1000
197
+ upload_last_n_checkpoints: 5
198
+ max_concurrent_uploads: 2
199
+ eval_strategy: 'no'
200
+ eval_steps: 500
201
+ eval_set_split_ratio: 0.1
202
+ eval_batch_size: 2
203
+ save_best_eval_metric_name: ''
204
+ save_best_eval_metric_greater_is_better: true
205
+ deepspeed_stage: 2
206
+ gradient_checkpointing: false
207
+ transformers_trust_remote_code: true
208
+ transformers_local_files_only: false
209
+ transformers_cache_dir: null
210
+ transformers_access_token: null
211
+ use_ddp: false
212
+ ddp_bucket_cap_mb: 100
213
+ num_gpus: 8
214
+ dataloader_num_workers: 8
215
+ remove_unused_columns: false
216
+ use_wandb: true
217
+ wandb_project: atomicvla
218
+ enable_profiling: false
219
+ max_retries: 3
220
+ assert_loss_less_than: null
221
+ add_rl_callback: false
222
+ enable_open_loop_eval: false
223
+ open_loop_eval_traj_ids:
224
+ - 0
225
+ open_loop_eval_steps_per_traj: 100
226
+ open_loop_eval_plot_indices: null
227
+ max_steps: 10000
228
+ save_steps: 2000
experiment_cfg/config.yaml ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ !!python/object:gr00t.configs.base_config.Config
2
+ data: !!python/object:gr00t.configs.data.data_config.DataConfig
3
+ allow_padding: false
4
+ datasets:
5
+ - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
6
+ dataset_paths:
7
+ - /fsx/ubuntu/atomicvla/data/robocasa365_subset_atomic_x2/v1.0/pretrain/atomic/CloseBlenderLid/20250822/lerobot
8
+ dataset_type: physical_embodiment
9
+ embodiment_tag: new_embodiment
10
+ mix_ratio: 1.0
11
+ val_dataset_path: null
12
+ download_cache: false
13
+ episode_sampling_rate: 0.1
14
+ image_crop_size:
15
+ - 244
16
+ - 244
17
+ image_target_size:
18
+ - 224
19
+ - 224
20
+ mock_dataset_mode: false
21
+ modality_configs:
22
+ new_embodiment:
23
+ action: !!python/object:gr00t.data.types.ModalityConfig
24
+ action_configs:
25
+ - !!python/object:gr00t.data.types.ActionConfig
26
+ format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
27
+ - default
28
+ rep: &id002 !!python/object/apply:gr00t.data.types.ActionRepresentation
29
+ - absolute
30
+ state_key: null
31
+ type: &id003 !!python/object/apply:gr00t.data.types.ActionType
32
+ - non_eef
33
+ - !!python/object:gr00t.data.types.ActionConfig
34
+ format: *id001
35
+ rep: *id002
36
+ state_key: null
37
+ type: *id003
38
+ - !!python/object:gr00t.data.types.ActionConfig
39
+ format: *id001
40
+ rep: *id002
41
+ state_key: null
42
+ type: &id004 !!python/object/apply:gr00t.data.types.ActionType
43
+ - eef
44
+ - !!python/object:gr00t.data.types.ActionConfig
45
+ format: *id001
46
+ rep: *id002
47
+ state_key: null
48
+ type: *id004
49
+ - !!python/object:gr00t.data.types.ActionConfig
50
+ format: *id001
51
+ rep: *id002
52
+ state_key: null
53
+ type: *id003
54
+ delta_indices:
55
+ - 0
56
+ - 1
57
+ - 2
58
+ - 3
59
+ - 4
60
+ - 5
61
+ - 6
62
+ - 7
63
+ - 8
64
+ - 9
65
+ - 10
66
+ - 11
67
+ - 12
68
+ - 13
69
+ - 14
70
+ - 15
71
+ mean_std_embedding_keys: null
72
+ modality_keys:
73
+ - base_motion
74
+ - control_mode
75
+ - end_effector_position
76
+ - end_effector_rotation
77
+ - gripper_close
78
+ sin_cos_embedding_keys: null
79
+ language: !!python/object:gr00t.data.types.ModalityConfig
80
+ action_configs: null
81
+ delta_indices:
82
+ - 0
83
+ mean_std_embedding_keys: null
84
+ modality_keys:
85
+ - sub_task
86
+ sin_cos_embedding_keys: null
87
+ state: !!python/object:gr00t.data.types.ModalityConfig
88
+ action_configs: null
89
+ delta_indices:
90
+ - 0
91
+ mean_std_embedding_keys: null
92
+ modality_keys:
93
+ - base_position
94
+ - base_rotation
95
+ - end_effector_position_relative
96
+ - end_effector_rotation_relative
97
+ - gripper_qpos
98
+ sin_cos_embedding_keys: null
99
+ video: !!python/object:gr00t.data.types.ModalityConfig
100
+ action_configs: null
101
+ delta_indices:
102
+ - 0
103
+ mean_std_embedding_keys: null
104
+ modality_keys:
105
+ - robot0_eye_in_hand
106
+ - robot0_agentview_left
107
+ - robot0_agentview_right
108
+ sin_cos_embedding_keys: null
109
+ mode: single_turn
110
+ multiprocessing_context: fork
111
+ num_shards_per_epoch: 100000
112
+ override_pretraining_statistics: false
113
+ random_chop: 0.0
114
+ seed: 42
115
+ shard_size: 1024
116
+ shuffle: true
117
+ subsample_ratio: 1.0
118
+ video_backend: torchcodec
119
+ load_config_path: null
120
+ model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
121
+ _attn_implementation_autoset: false
122
+ _attn_implementation_internal: null
123
+ _commit_hash: null
124
+ _name_or_path: ''
125
+ add_cross_attention: false
126
+ architectures: null
127
+ backbone_model_type: eagle
128
+ backbone_trainable_params_fp32: true
129
+ bad_words_ids: null
130
+ begin_suppress_tokens: null
131
+ bos_token_id: null
132
+ chunk_size_feed_forward: 0
133
+ cross_attention_hidden_size: null
134
+ decoder_start_token_id: null
135
+ diffusion_model_cfg:
136
+ attention_head_dim: 48
137
+ dropout: 0.2
138
+ final_dropout: true
139
+ interleave_self_attention: true
140
+ norm_type: ada_norm
141
+ num_attention_heads: 32
142
+ num_layers: 32
143
+ output_dim: 1024
144
+ positional_embeddings: null
145
+ diversity_penalty: 0.0
146
+ do_sample: false
147
+ eagle_collator: true
148
+ early_stopping: false
149
+ encoder_no_repeat_ngram_size: 0
150
+ eos_token_id: null
151
+ exponential_decay_length_penalty: null
152
+ finetuning_task: null
153
+ forced_bos_token_id: null
154
+ forced_eos_token_id: null
155
+ id2label:
156
+ 0: LABEL_0
157
+ 1: LABEL_1
158
+ is_decoder: false
159
+ is_encoder_decoder: false
160
+ label2id:
161
+ LABEL_0: 0
162
+ LABEL_1: 1
163
+ length_penalty: 1.0
164
+ load_bf16: true
165
+ max_length: 20
166
+ min_length: 0
167
+ model_name: nvidia/Eagle-Block2A-2B-v2
168
+ no_repeat_ngram_size: 0
169
+ num_beam_groups: 1
170
+ num_beams: 1
171
+ num_return_sequences: 1
172
+ output_attentions: false
173
+ output_hidden_states: false
174
+ output_scores: false
175
+ pad_token_id: null
176
+ prefix: null
177
+ problem_type: null
178
+ pruned_heads: {}
179
+ remove_invalid_values: false
180
+ repetition_penalty: 1.0
181
+ reproject_vision: false
182
+ return_dict: true
183
+ return_dict_in_generate: false
184
+ sep_token_id: null
185
+ suppress_tokens: null
186
+ task_specific_params: null
187
+ temperature: 1.0
188
+ tf_legacy_loss: false
189
+ tie_encoder_decoder: false
190
+ tie_word_embeddings: true
191
+ tokenizer_class: null
192
+ top_k: 50
193
+ top_p: 1.0
194
+ torch_dtype: null
195
+ torchscript: false
196
+ transformers_version: null
197
+ typical_p: 1.0
198
+ use_bfloat16: false
199
+ use_relative_action: true
200
+ training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
201
+ add_rl_callback: false
202
+ assert_loss_less_than: null
203
+ batch_size: null
204
+ bf16: true
205
+ dataloader_num_workers: 8
206
+ ddp_bucket_cap_mb: 100
207
+ deepspeed_stage: 2
208
+ enable_open_loop_eval: false
209
+ enable_profiling: false
210
+ eval_batch_size: 2
211
+ eval_bf16: true
212
+ eval_set_split_ratio: 0.1
213
+ eval_steps: 500
214
+ eval_strategy: 'no'
215
+ experiment_name: null
216
+ fp16: false
217
+ global_batch_size: 128
218
+ gradient_accumulation_steps: 1
219
+ gradient_checkpointing: false
220
+ learning_rate: 0.0001
221
+ logging_steps: 10
222
+ lr_scheduler_type: cosine
223
+ max_concurrent_uploads: 2
224
+ max_grad_norm: 1.0
225
+ max_retries: 3
226
+ max_steps: 10000
227
+ num_gpus: 8
228
+ open_loop_eval_plot_indices: null
229
+ open_loop_eval_steps_per_traj: 100
230
+ open_loop_eval_traj_ids:
231
+ - 0
232
+ optim: adamw_torch
233
+ output_dir: ./ckpt/abcd2a_control_ft
234
+ remove_unused_columns: false
235
+ reset_optimizer: false
236
+ save_best_eval_metric_greater_is_better: true
237
+ save_best_eval_metric_name: ''
238
+ save_steps: 2000
239
+ save_total_limit: 5
240
+ save_vl_model: false
241
+ start_from_checkpoint: ./ckpt/abcd2a_control_pt
242
+ tf32: true
243
+ transformers_access_token: null
244
+ transformers_cache_dir: null
245
+ transformers_local_files_only: false
246
+ transformers_trust_remote_code: true
247
+ upload_checkpoints: false
248
+ upload_every: 1000
249
+ upload_last_n_checkpoints: 5
250
+ use_ddp: false
251
+ use_wandb: true
252
+ wandb_project: atomicvla
253
+ warmup_ratio: 0.05
254
+ warmup_steps: 0
255
+ weight_decay: 1.0e-05
256
+ wsd_decay_steps: 3000
257
+ wsd_decay_type: cosine
258
+ wsd_min_lr_ratio: 0.0
experiment_cfg/dataset_statistics.json ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "new_embodiment": {
3
+ "state": {
4
+ "base_position": {
5
+ "min": [
6
+ -2.275968074798584,
7
+ -5.743692398071289,
8
+ 0.6999539136886597
9
+ ],
10
+ "max": [
11
+ 5.842284202575684,
12
+ -0.3689787983894348,
13
+ 0.7079945802688599
14
+ ],
15
+ "mean": [
16
+ 2.74623703956604,
17
+ -2.616032838821411,
18
+ 0.7009382247924805
19
+ ],
20
+ "std": [
21
+ 1.5239065885543823,
22
+ 1.2950901985168457,
23
+ 0.0012742063263423056
24
+ ],
25
+ "q01": [
26
+ -2.2121946811676025,
27
+ -5.54568338394165,
28
+ 0.6999894976615906
29
+ ],
30
+ "q99": [
31
+ 5.50187873840332,
32
+ -0.37999117374420166,
33
+ 0.7055556178092957
34
+ ]
35
+ },
36
+ "base_rotation": {
37
+ "min": [
38
+ 0.0,
39
+ 0.0,
40
+ -1.0,
41
+ 0.0
42
+ ],
43
+ "max": [
44
+ 0.0,
45
+ 0.0,
46
+ 1.0,
47
+ 1.0
48
+ ],
49
+ "mean": [
50
+ 0.0,
51
+ 0.0,
52
+ -0.09249360859394073,
53
+ 0.4838332235813141
54
+ ],
55
+ "std": [
56
+ 0.0,
57
+ 0.0,
58
+ 0.7788283228874207,
59
+ 0.3881465494632721
60
+ ],
61
+ "q01": [
62
+ 0.0,
63
+ 0.0,
64
+ -1.0,
65
+ 3.4066431453538826e-07
66
+ ],
67
+ "q99": [
68
+ 0.0,
69
+ 0.0,
70
+ 1.0,
71
+ 1.0
72
+ ]
73
+ },
74
+ "end_effector_position_relative": {
75
+ "min": [
76
+ -0.07487241923809052,
77
+ -0.5990239381790161,
78
+ 0.22885531187057495
79
+ ],
80
+ "max": [
81
+ 0.49205079674720764,
82
+ 0.6436583399772644,
83
+ 0.8710702657699585
84
+ ],
85
+ "mean": [
86
+ 0.2536119520664215,
87
+ -0.003847635118290782,
88
+ 0.550906240940094
89
+ ],
90
+ "std": [
91
+ 0.07798480987548828,
92
+ 0.22548064589500427,
93
+ 0.17645911872386932
94
+ ],
95
+ "q01": [
96
+ 0.044332973659038544,
97
+ -0.496540904045105,
98
+ 0.24020594358444214
99
+ ],
100
+ "q99": [
101
+ 0.39605388045310974,
102
+ 0.48453831672668457,
103
+ 0.7898283004760742
104
+ ]
105
+ },
106
+ "end_effector_rotation_relative": {
107
+ "min": [
108
+ -0.9999930262565613,
109
+ -0.9147764444351196,
110
+ -0.7121668457984924,
111
+ 3.4656542879929475e-07
112
+ ],
113
+ "max": [
114
+ 0.9999456405639648,
115
+ 0.9148962497711182,
116
+ 0.6037217378616333,
117
+ 0.6980910897254944
118
+ ],
119
+ "mean": [
120
+ -0.19048655033111572,
121
+ 0.1259855329990387,
122
+ -0.020019982010126114,
123
+ 0.09044083952903748
124
+ ],
125
+ "std": [
126
+ 0.8809337615966797,
127
+ 0.32437989115715027,
128
+ 0.21401339769363403,
129
+ 0.11041491478681564
130
+ ],
131
+ "q01": [
132
+ -0.9987776875495911,
133
+ -0.7081342935562134,
134
+ -0.5060954093933105,
135
+ 0.000825436320155859
136
+ ],
137
+ "q99": [
138
+ 0.9972664713859558,
139
+ 0.7635475397109985,
140
+ 0.5001609921455383,
141
+ 0.5715427398681641
142
+ ]
143
+ },
144
+ "gripper_qpos": {
145
+ "min": [
146
+ -0.0020662746392190456,
147
+ -0.04189836606383324
148
+ ],
149
+ "max": [
150
+ 0.04232558235526085,
151
+ 0.0006283874390646815
152
+ ],
153
+ "mean": [
154
+ 0.032970815896987915,
155
+ -0.03249610587954521
156
+ ],
157
+ "std": [
158
+ 0.009412867948412895,
159
+ 0.009693702682852745
160
+ ],
161
+ "q01": [
162
+ 0.007987073622643948,
163
+ -0.040516264736652374
164
+ ],
165
+ "q99": [
166
+ 0.04054218530654907,
167
+ -0.007861155085265636
168
+ ]
169
+ }
170
+ },
171
+ "action": {
172
+ "base_motion": {
173
+ "min": [
174
+ -1.0,
175
+ -1.0,
176
+ -0.19142857193946838,
177
+ 0.0
178
+ ],
179
+ "max": [
180
+ 0.8642857074737549,
181
+ 1.0,
182
+ 0.1599999964237213,
183
+ 0.0
184
+ ],
185
+ "mean": [
186
+ 0.0015367598971351981,
187
+ 0.0011513116769492626,
188
+ 2.8855352866230533e-05,
189
+ 0.0
190
+ ],
191
+ "std": [
192
+ 0.057876817882061005,
193
+ 0.06887566298246384,
194
+ 0.006419945973902941,
195
+ 0.0
196
+ ],
197
+ "q01": [
198
+ 0.0,
199
+ 0.0,
200
+ 0.0,
201
+ 0.0
202
+ ],
203
+ "q99": [
204
+ 0.05000000074505806,
205
+ 0.0,
206
+ 0.0,
207
+ 0.0
208
+ ]
209
+ },
210
+ "control_mode": {
211
+ "min": [
212
+ -1.0
213
+ ],
214
+ "max": [
215
+ 1.0
216
+ ],
217
+ "mean": [
218
+ -0.9610646367073059
219
+ ],
220
+ "std": [
221
+ 0.27625617384910583
222
+ ],
223
+ "q01": [
224
+ -1.0
225
+ ],
226
+ "q99": [
227
+ 1.0
228
+ ]
229
+ },
230
+ "end_effector_position": {
231
+ "min": [
232
+ -1.0,
233
+ -1.0,
234
+ -1.0
235
+ ],
236
+ "max": [
237
+ 1.0,
238
+ 1.0,
239
+ 1.0
240
+ ],
241
+ "mean": [
242
+ -0.03847881779074669,
243
+ 0.010149438865482807,
244
+ 0.009745091199874878
245
+ ],
246
+ "std": [
247
+ 0.29936665296554565,
248
+ 0.34878847002983093,
249
+ 0.3984452784061432
250
+ ],
251
+ "q01": [
252
+ -0.9928571581840515,
253
+ -1.0,
254
+ -0.7571428418159485
255
+ ],
256
+ "q99": [
257
+ 0.5785714387893677,
258
+ 1.0,
259
+ 1.0
260
+ ]
261
+ },
262
+ "end_effector_rotation": {
263
+ "min": [
264
+ -0.345714271068573,
265
+ -0.5571428537368774,
266
+ -0.8342857360839844
267
+ ],
268
+ "max": [
269
+ 0.44857141375541687,
270
+ 0.46857142448425293,
271
+ 0.7942857146263123
272
+ ],
273
+ "mean": [
274
+ 0.002458197297528386,
275
+ -0.013575965538620949,
276
+ 0.001469695707783103
277
+ ],
278
+ "std": [
279
+ 0.05701429396867752,
280
+ 0.10766582936048508,
281
+ 0.08917795866727829
282
+ ],
283
+ "q01": [
284
+ -0.17428570985794067,
285
+ -0.31142857670783997,
286
+ -0.35428571701049805
287
+ ],
288
+ "q99": [
289
+ 0.2085714340209961,
290
+ 0.27714285254478455,
291
+ 0.27714285254478455
292
+ ]
293
+ },
294
+ "gripper_close": {
295
+ "min": [
296
+ -1.0
297
+ ],
298
+ "max": [
299
+ 1.0
300
+ ],
301
+ "mean": [
302
+ 0.02607424184679985
303
+ ],
304
+ "std": [
305
+ 0.9997770190238954
306
+ ],
307
+ "q01": [
308
+ -1.0
309
+ ],
310
+ "q99": [
311
+ 1.0
312
+ ]
313
+ }
314
+ },
315
+ "relative_action": {}
316
+ }
317
+ }
experiment_cfg/final_model_config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "Gr00tN1d6",
3
+ "model_dtype": "bfloat16",
4
+ "model_name": "nvidia/Eagle-Block2A-2B-v2",
5
+ "backbone_model_type": "eagle",
6
+ "model_revision": null,
7
+ "tune_top_llm_layers": 4,
8
+ "backbone_embedding_dim": 2048,
9
+ "tune_llm": false,
10
+ "tune_visual": false,
11
+ "select_layer": 16,
12
+ "reproject_vision": false,
13
+ "use_flash_attention": true,
14
+ "load_bf16": true,
15
+ "collator_overwrite_image_inputs": false,
16
+ "eagle_collator": true,
17
+ "backbone_trainable_params_fp32": true,
18
+ "apply_sincos_state_encoding": true,
19
+ "use_relative_action": true,
20
+ "max_state_dim": 128,
21
+ "max_action_dim": 128,
22
+ "action_horizon": 50,
23
+ "hidden_size": 1024,
24
+ "input_embedding_dim": 1536,
25
+ "add_pos_embed": true,
26
+ "attn_dropout": 0.2,
27
+ "use_vlln": true,
28
+ "max_seq_len": 1024,
29
+ "use_alternate_vl_dit": true,
30
+ "attend_text_every_n_blocks": 2,
31
+ "diffusion_model_cfg": {
32
+ "attention_head_dim": 48,
33
+ "dropout": 0.2,
34
+ "final_dropout": true,
35
+ "interleave_self_attention": true,
36
+ "norm_type": "ada_norm",
37
+ "num_attention_heads": 32,
38
+ "num_layers": 32,
39
+ "output_dim": 1024,
40
+ "positional_embeddings": null
41
+ },
42
+ "num_inference_timesteps": 4,
43
+ "noise_beta_alpha": 1.5,
44
+ "noise_beta_beta": 1.0,
45
+ "noise_s": 0.999,
46
+ "num_timestep_buckets": 1000,
47
+ "tune_projector": true,
48
+ "tune_diffusion_model": true,
49
+ "tune_vlln": true,
50
+ "state_dropout_prob": 0.0,
51
+ "state_additive_noise_scale": 0.0,
52
+ "max_num_embodiments": 32
53
+ }
experiment_cfg/final_processor_config.json ADDED
The diff for this file is too large to render. See raw diff
 
global_step6000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:daa5ea2e2d5c42cf64b0a01b1b85478c01b1b169889e103fc749fad3d0c3af32
3
+ size 2429964677
global_step6000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb55b6a5aa5040e86a92165cdae1a540401524d4a5dba7490168ebef90c4c661
3
+ size 2429964421
global_step6000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc018b38ea96a5f9d2af7013a288117b13c08061a2c09ec9a33caaba2a78f171
3
+ size 2429964613
global_step6000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f062854744ae7ae6e1d66300734bb8986fa767d7986493109781f4bbba2d2da6
3
+ size 2429964485
global_step6000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59a4c10d1af52e835b1d7c583a308d094d29e11e376de363e55aa371dcb1bc2e
3
+ size 2429964421
global_step6000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f115b2bc9b0e543b27bc8fa036f206e2d23abcf29c57021f306cc64f4204538b
3
+ size 2429964613
global_step6000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58a801614aa619907dc32f5cf7b948fe762a7b0554541c7f02b68fd96f0596f8
3
+ size 2429962437
global_step6000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97a9651e51fa08c1853e2e9d287c6fc41006b6d9f2cfdfb43615523bec98a8af
3
+ size 2429960773
global_step6000/mp_rank_00_model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59c65251f10b19a69e6b30d1e6d44593a55e4cb89a6c7f466134c2cfd087fae9
3
+ size 9907202435
latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step6000
model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6aad51efbdb9e086977f366caaeb17934feb50452e16187366d675b8f3e92daa
3
+ size 4991091456
model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3619ed842fce487bba88bfafdab5aa0584b9950388475b0c1098477b9a23a56
3
+ size 1582283096
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
processor_config.json ADDED
@@ -0,0 +1,480 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "processor_class": "Gr00tN1d6Processor",
3
+ "processor_kwargs": {
4
+ "modality_configs": {
5
+ "behavior_r1_pro": {
6
+ "video": {
7
+ "delta_indices": [
8
+ 0
9
+ ],
10
+ "modality_keys": [
11
+ "observation.images.rgb.head_256_256",
12
+ "observation.images.rgb.left_wrist_256_256",
13
+ "observation.images.rgb.right_wrist_256_256"
14
+ ],
15
+ "sin_cos_embedding_keys": null,
16
+ "mean_std_embedding_keys": null,
17
+ "action_configs": null
18
+ },
19
+ "state": {
20
+ "delta_indices": [
21
+ 0
22
+ ],
23
+ "modality_keys": [
24
+ "robot_pos",
25
+ "robot_ori_cos",
26
+ "robot_ori_sin",
27
+ "robot_2d_ori",
28
+ "robot_2d_ori_cos",
29
+ "robot_2d_ori_sin",
30
+ "robot_lin_vel",
31
+ "robot_ang_vel",
32
+ "arm_left_qpos",
33
+ "arm_left_qpos_sin",
34
+ "arm_left_qpos_cos",
35
+ "eef_left_pos",
36
+ "eef_left_quat",
37
+ "gripper_left_qpos",
38
+ "arm_right_qpos",
39
+ "arm_right_qpos_sin",
40
+ "arm_right_qpos_cos",
41
+ "eef_right_pos",
42
+ "eef_right_quat",
43
+ "gripper_right_qpos",
44
+ "trunk_qpos"
45
+ ],
46
+ "sin_cos_embedding_keys": null,
47
+ "mean_std_embedding_keys": null,
48
+ "action_configs": null
49
+ },
50
+ "action": {
51
+ "delta_indices": [
52
+ 0,
53
+ 1,
54
+ 2,
55
+ 3,
56
+ 4,
57
+ 5,
58
+ 6,
59
+ 7,
60
+ 8,
61
+ 9,
62
+ 10,
63
+ 11,
64
+ 12,
65
+ 13,
66
+ 14,
67
+ 15,
68
+ 16,
69
+ 17,
70
+ 18,
71
+ 19,
72
+ 20,
73
+ 21,
74
+ 22,
75
+ 23,
76
+ 24,
77
+ 25,
78
+ 26,
79
+ 27,
80
+ 28,
81
+ 29,
82
+ 30,
83
+ 31
84
+ ],
85
+ "modality_keys": [
86
+ "base",
87
+ "torso",
88
+ "left_arm",
89
+ "left_gripper",
90
+ "right_arm",
91
+ "right_gripper"
92
+ ],
93
+ "sin_cos_embedding_keys": null,
94
+ "mean_std_embedding_keys": null,
95
+ "action_configs": [
96
+ {
97
+ "rep": "ABSOLUTE",
98
+ "type": "NON_EEF",
99
+ "format": "DEFAULT",
100
+ "state_key": null
101
+ },
102
+ {
103
+ "rep": "RELATIVE",
104
+ "type": "NON_EEF",
105
+ "format": "DEFAULT",
106
+ "state_key": "trunk_qpos"
107
+ },
108
+ {
109
+ "rep": "RELATIVE",
110
+ "type": "NON_EEF",
111
+ "format": "DEFAULT",
112
+ "state_key": "arm_left_qpos"
113
+ },
114
+ {
115
+ "rep": "ABSOLUTE",
116
+ "type": "NON_EEF",
117
+ "format": "DEFAULT",
118
+ "state_key": null
119
+ },
120
+ {
121
+ "rep": "RELATIVE",
122
+ "type": "NON_EEF",
123
+ "format": "DEFAULT",
124
+ "state_key": "arm_right_qpos"
125
+ },
126
+ {
127
+ "rep": "ABSOLUTE",
128
+ "type": "NON_EEF",
129
+ "format": "DEFAULT",
130
+ "state_key": null
131
+ }
132
+ ]
133
+ },
134
+ "language": {
135
+ "delta_indices": [
136
+ 0
137
+ ],
138
+ "modality_keys": [
139
+ "annotation.human.coarse_action"
140
+ ],
141
+ "sin_cos_embedding_keys": null,
142
+ "mean_std_embedding_keys": null,
143
+ "action_configs": null
144
+ }
145
+ },
146
+ "gr1": {
147
+ "video": {
148
+ "delta_indices": [
149
+ 0
150
+ ],
151
+ "modality_keys": [
152
+ "ego_view_bg_crop_pad_res256_freq20"
153
+ ],
154
+ "sin_cos_embedding_keys": null,
155
+ "mean_std_embedding_keys": null,
156
+ "action_configs": null
157
+ },
158
+ "state": {
159
+ "delta_indices": [
160
+ 0
161
+ ],
162
+ "modality_keys": [
163
+ "left_arm",
164
+ "right_arm",
165
+ "left_hand",
166
+ "right_hand",
167
+ "waist"
168
+ ],
169
+ "sin_cos_embedding_keys": [
170
+ "left_arm",
171
+ "right_arm",
172
+ "left_hand",
173
+ "right_hand",
174
+ "waist"
175
+ ],
176
+ "mean_std_embedding_keys": null,
177
+ "action_configs": null
178
+ },
179
+ "action": {
180
+ "delta_indices": [
181
+ 0,
182
+ 1,
183
+ 2,
184
+ 3,
185
+ 4,
186
+ 5,
187
+ 6,
188
+ 7,
189
+ 8,
190
+ 9,
191
+ 10,
192
+ 11,
193
+ 12,
194
+ 13,
195
+ 14,
196
+ 15
197
+ ],
198
+ "modality_keys": [
199
+ "left_arm",
200
+ "right_arm",
201
+ "left_hand",
202
+ "right_hand",
203
+ "waist"
204
+ ],
205
+ "sin_cos_embedding_keys": null,
206
+ "mean_std_embedding_keys": null,
207
+ "action_configs": [
208
+ {
209
+ "rep": "RELATIVE",
210
+ "type": "NON_EEF",
211
+ "format": "DEFAULT",
212
+ "state_key": null
213
+ },
214
+ {
215
+ "rep": "RELATIVE",
216
+ "type": "NON_EEF",
217
+ "format": "DEFAULT",
218
+ "state_key": null
219
+ },
220
+ {
221
+ "rep": "RELATIVE",
222
+ "type": "NON_EEF",
223
+ "format": "DEFAULT",
224
+ "state_key": null
225
+ },
226
+ {
227
+ "rep": "RELATIVE",
228
+ "type": "NON_EEF",
229
+ "format": "DEFAULT",
230
+ "state_key": null
231
+ },
232
+ {
233
+ "rep": "ABSOLUTE",
234
+ "type": "NON_EEF",
235
+ "format": "DEFAULT",
236
+ "state_key": null
237
+ }
238
+ ]
239
+ },
240
+ "language": {
241
+ "delta_indices": [
242
+ 0
243
+ ],
244
+ "modality_keys": [
245
+ "task"
246
+ ],
247
+ "sin_cos_embedding_keys": null,
248
+ "mean_std_embedding_keys": null,
249
+ "action_configs": null
250
+ }
251
+ },
252
+ "robocasa_panda_omron": {
253
+ "video": {
254
+ "delta_indices": [
255
+ 0
256
+ ],
257
+ "modality_keys": [
258
+ "res256_image_side_0",
259
+ "res256_image_side_1",
260
+ "res256_image_wrist_0"
261
+ ],
262
+ "sin_cos_embedding_keys": null,
263
+ "mean_std_embedding_keys": null,
264
+ "action_configs": null
265
+ },
266
+ "state": {
267
+ "delta_indices": [
268
+ 0
269
+ ],
270
+ "modality_keys": [
271
+ "end_effector_position_relative",
272
+ "end_effector_rotation_relative",
273
+ "gripper_qpos",
274
+ "base_position",
275
+ "base_rotation"
276
+ ],
277
+ "sin_cos_embedding_keys": null,
278
+ "mean_std_embedding_keys": null,
279
+ "action_configs": null
280
+ },
281
+ "action": {
282
+ "delta_indices": [
283
+ 0,
284
+ 1,
285
+ 2,
286
+ 3,
287
+ 4,
288
+ 5,
289
+ 6,
290
+ 7,
291
+ 8,
292
+ 9,
293
+ 10,
294
+ 11,
295
+ 12,
296
+ 13,
297
+ 14,
298
+ 15
299
+ ],
300
+ "modality_keys": [
301
+ "end_effector_position",
302
+ "end_effector_rotation",
303
+ "gripper_close",
304
+ "base_motion",
305
+ "control_mode"
306
+ ],
307
+ "sin_cos_embedding_keys": null,
308
+ "mean_std_embedding_keys": null,
309
+ "action_configs": [
310
+ {
311
+ "rep": "ABSOLUTE",
312
+ "type": "NON_EEF",
313
+ "format": "DEFAULT",
314
+ "state_key": null
315
+ },
316
+ {
317
+ "rep": "ABSOLUTE",
318
+ "type": "NON_EEF",
319
+ "format": "DEFAULT",
320
+ "state_key": null
321
+ },
322
+ {
323
+ "rep": "ABSOLUTE",
324
+ "type": "NON_EEF",
325
+ "format": "DEFAULT",
326
+ "state_key": null
327
+ },
328
+ {
329
+ "rep": "ABSOLUTE",
330
+ "type": "NON_EEF",
331
+ "format": "DEFAULT",
332
+ "state_key": null
333
+ },
334
+ {
335
+ "rep": "ABSOLUTE",
336
+ "type": "NON_EEF",
337
+ "format": "DEFAULT",
338
+ "state_key": null
339
+ }
340
+ ]
341
+ },
342
+ "language": {
343
+ "delta_indices": [
344
+ 0
345
+ ],
346
+ "modality_keys": [
347
+ "annotation.human.action.task_description"
348
+ ],
349
+ "sin_cos_embedding_keys": null,
350
+ "mean_std_embedding_keys": null,
351
+ "action_configs": null
352
+ }
353
+ },
354
+ "new_embodiment": {
355
+ "video": {
356
+ "delta_indices": [
357
+ 0
358
+ ],
359
+ "modality_keys": [
360
+ "robot0_eye_in_hand",
361
+ "robot0_agentview_left",
362
+ "robot0_agentview_right"
363
+ ],
364
+ "sin_cos_embedding_keys": null,
365
+ "mean_std_embedding_keys": null,
366
+ "action_configs": null
367
+ },
368
+ "state": {
369
+ "delta_indices": [
370
+ 0
371
+ ],
372
+ "modality_keys": [
373
+ "base_position",
374
+ "base_rotation",
375
+ "end_effector_position_relative",
376
+ "end_effector_rotation_relative",
377
+ "gripper_qpos"
378
+ ],
379
+ "sin_cos_embedding_keys": null,
380
+ "mean_std_embedding_keys": null,
381
+ "action_configs": null
382
+ },
383
+ "action": {
384
+ "delta_indices": [
385
+ 0,
386
+ 1,
387
+ 2,
388
+ 3,
389
+ 4,
390
+ 5,
391
+ 6,
392
+ 7,
393
+ 8,
394
+ 9,
395
+ 10,
396
+ 11,
397
+ 12,
398
+ 13,
399
+ 14,
400
+ 15
401
+ ],
402
+ "modality_keys": [
403
+ "base_motion",
404
+ "control_mode",
405
+ "end_effector_position",
406
+ "end_effector_rotation",
407
+ "gripper_close"
408
+ ],
409
+ "sin_cos_embedding_keys": null,
410
+ "mean_std_embedding_keys": null,
411
+ "action_configs": [
412
+ {
413
+ "rep": "ABSOLUTE",
414
+ "type": "NON_EEF",
415
+ "format": "DEFAULT",
416
+ "state_key": null
417
+ },
418
+ {
419
+ "rep": "ABSOLUTE",
420
+ "type": "NON_EEF",
421
+ "format": "DEFAULT",
422
+ "state_key": null
423
+ },
424
+ {
425
+ "rep": "ABSOLUTE",
426
+ "type": "EEF",
427
+ "format": "DEFAULT",
428
+ "state_key": null
429
+ },
430
+ {
431
+ "rep": "ABSOLUTE",
432
+ "type": "EEF",
433
+ "format": "DEFAULT",
434
+ "state_key": null
435
+ },
436
+ {
437
+ "rep": "ABSOLUTE",
438
+ "type": "NON_EEF",
439
+ "format": "DEFAULT",
440
+ "state_key": null
441
+ }
442
+ ]
443
+ },
444
+ "language": {
445
+ "delta_indices": [
446
+ 0
447
+ ],
448
+ "modality_keys": [
449
+ "sub_task"
450
+ ],
451
+ "sin_cos_embedding_keys": null,
452
+ "mean_std_embedding_keys": null,
453
+ "action_configs": null
454
+ }
455
+ }
456
+ },
457
+ "image_crop_size": null,
458
+ "image_target_size": null,
459
+ "use_albumentations": true,
460
+ "random_rotation_angle": null,
461
+ "color_jitter_params": {
462
+ "brightness": 0.3,
463
+ "contrast": 0.4,
464
+ "saturation": 0.5,
465
+ "hue": 0.08
466
+ },
467
+ "shortest_image_edge": 256,
468
+ "crop_fraction": 0.95,
469
+ "model_name": "nvidia/Eagle-Block2A-2B-v2",
470
+ "model_type": "eagle",
471
+ "formalize_language": true,
472
+ "max_state_dim": 128,
473
+ "max_action_dim": 128,
474
+ "max_action_horizon": 50,
475
+ "use_percentiles": false,
476
+ "clip_outliers": true,
477
+ "apply_sincos_state_encoding": true,
478
+ "use_relative_action": true
479
+ }
480
+ }
rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:339a9adc3b579f192e4ee6cd5cade0f4998295970e72d6e67ccf819a3b728244
3
+ size 16389
rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad2746ca4c9083f38631a3c59ffed239252180743c4d7177e0b0c52e06be71a5
3
+ size 16389
rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca33beedbcd04df9fe66ab716a11b71900a4c4e79865bfa4b29be9fe078d40e0
3
+ size 16389
rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:968614ae6b34852c39d23b58e45d1609c897b60d32be9299655ebd9b6ec9bbaf
3
+ size 16389
rng_state_4.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3e002c8639a24265f41c5463950b7d16382069b86b8dfbad08784db8d2c1c94
3
+ size 16389
rng_state_5.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3e16d2db5723745dca2d24af97a5ef39fe343ec08ab3932b2f42c67df13f9253
3
+ size 16389
rng_state_6.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05213087cda53a1940369b85f37d4c5778a800430e6c294d624507d9a04ceca8
3
+ size 16389
rng_state_7.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:001b7e431843661d8e1ed2f11e580856f3b81c78125eff519ed1bb1f8b58216a
3
+ size 16389
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c81168c968205dd4f18f99afb8d250e3039fa2276a497474189eec84cb61b541
3
+ size 1465
statistics.json ADDED
The diff for this file is too large to render. See raw diff
 
trainer_state.json ADDED
@@ -0,0 +1,3634 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.6,
6
+ "eval_steps": 500,
7
+ "global_step": 6000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "grad_norm": 0.5420440435409546,
14
+ "learning_rate": 1.8e-06,
15
+ "loss": 0.0694,
16
+ "step": 10
17
+ },
18
+ {
19
+ "grad_norm": 0.7189459204673767,
20
+ "learning_rate": 3.8e-06,
21
+ "loss": 0.0645,
22
+ "step": 20
23
+ },
24
+ {
25
+ "grad_norm": 0.21701538562774658,
26
+ "learning_rate": 5.8e-06,
27
+ "loss": 0.0317,
28
+ "step": 30
29
+ },
30
+ {
31
+ "grad_norm": 0.29029083251953125,
32
+ "learning_rate": 7.8e-06,
33
+ "loss": 0.0315,
34
+ "step": 40
35
+ },
36
+ {
37
+ "grad_norm": 0.10109149664640427,
38
+ "learning_rate": 9.800000000000001e-06,
39
+ "loss": 0.0273,
40
+ "step": 50
41
+ },
42
+ {
43
+ "grad_norm": 0.19640426337718964,
44
+ "learning_rate": 1.18e-05,
45
+ "loss": 0.0319,
46
+ "step": 60
47
+ },
48
+ {
49
+ "grad_norm": 0.1066979169845581,
50
+ "learning_rate": 1.3800000000000002e-05,
51
+ "loss": 0.0251,
52
+ "step": 70
53
+ },
54
+ {
55
+ "grad_norm": 0.12810614705085754,
56
+ "learning_rate": 1.58e-05,
57
+ "loss": 0.024,
58
+ "step": 80
59
+ },
60
+ {
61
+ "grad_norm": 0.08170574903488159,
62
+ "learning_rate": 1.78e-05,
63
+ "loss": 0.0232,
64
+ "step": 90
65
+ },
66
+ {
67
+ "grad_norm": 0.16492632031440735,
68
+ "learning_rate": 1.9800000000000004e-05,
69
+ "loss": 0.0283,
70
+ "step": 100
71
+ },
72
+ {
73
+ "grad_norm": 0.14535479247570038,
74
+ "learning_rate": 2.18e-05,
75
+ "loss": 0.0231,
76
+ "step": 110
77
+ },
78
+ {
79
+ "grad_norm": 0.17147088050842285,
80
+ "learning_rate": 2.38e-05,
81
+ "loss": 0.0315,
82
+ "step": 120
83
+ },
84
+ {
85
+ "grad_norm": 0.10768885165452957,
86
+ "learning_rate": 2.58e-05,
87
+ "loss": 0.0194,
88
+ "step": 130
89
+ },
90
+ {
91
+ "grad_norm": 0.10331500321626663,
92
+ "learning_rate": 2.7800000000000005e-05,
93
+ "loss": 0.0203,
94
+ "step": 140
95
+ },
96
+ {
97
+ "grad_norm": 0.15777619183063507,
98
+ "learning_rate": 2.98e-05,
99
+ "loss": 0.0285,
100
+ "step": 150
101
+ },
102
+ {
103
+ "grad_norm": 0.11859080195426941,
104
+ "learning_rate": 3.18e-05,
105
+ "loss": 0.0234,
106
+ "step": 160
107
+ },
108
+ {
109
+ "grad_norm": 0.20379579067230225,
110
+ "learning_rate": 3.38e-05,
111
+ "loss": 0.0246,
112
+ "step": 170
113
+ },
114
+ {
115
+ "grad_norm": 0.08920297771692276,
116
+ "learning_rate": 3.58e-05,
117
+ "loss": 0.0209,
118
+ "step": 180
119
+ },
120
+ {
121
+ "grad_norm": 0.11533823609352112,
122
+ "learning_rate": 3.7800000000000004e-05,
123
+ "loss": 0.0261,
124
+ "step": 190
125
+ },
126
+ {
127
+ "grad_norm": 0.1448919177055359,
128
+ "learning_rate": 3.9800000000000005e-05,
129
+ "loss": 0.0264,
130
+ "step": 200
131
+ },
132
+ {
133
+ "grad_norm": 0.1489187777042389,
134
+ "learning_rate": 4.18e-05,
135
+ "loss": 0.0249,
136
+ "step": 210
137
+ },
138
+ {
139
+ "grad_norm": 0.1333455890417099,
140
+ "learning_rate": 4.38e-05,
141
+ "loss": 0.0281,
142
+ "step": 220
143
+ },
144
+ {
145
+ "grad_norm": 0.09986919909715652,
146
+ "learning_rate": 4.58e-05,
147
+ "loss": 0.0241,
148
+ "step": 230
149
+ },
150
+ {
151
+ "grad_norm": 0.15185418725013733,
152
+ "learning_rate": 4.78e-05,
153
+ "loss": 0.0279,
154
+ "step": 240
155
+ },
156
+ {
157
+ "grad_norm": 0.16237834095954895,
158
+ "learning_rate": 4.9800000000000004e-05,
159
+ "loss": 0.0294,
160
+ "step": 250
161
+ },
162
+ {
163
+ "grad_norm": 0.25100767612457275,
164
+ "learning_rate": 5.1800000000000005e-05,
165
+ "loss": 0.0292,
166
+ "step": 260
167
+ },
168
+ {
169
+ "grad_norm": 0.1342630535364151,
170
+ "learning_rate": 5.380000000000001e-05,
171
+ "loss": 0.0291,
172
+ "step": 270
173
+ },
174
+ {
175
+ "grad_norm": 0.1496191918849945,
176
+ "learning_rate": 5.580000000000001e-05,
177
+ "loss": 0.0332,
178
+ "step": 280
179
+ },
180
+ {
181
+ "grad_norm": 0.1313694268465042,
182
+ "learning_rate": 5.7799999999999995e-05,
183
+ "loss": 0.0251,
184
+ "step": 290
185
+ },
186
+ {
187
+ "grad_norm": 0.13200165331363678,
188
+ "learning_rate": 5.9800000000000003e-05,
189
+ "loss": 0.0307,
190
+ "step": 300
191
+ },
192
+ {
193
+ "grad_norm": 0.16884654760360718,
194
+ "learning_rate": 6.18e-05,
195
+ "loss": 0.0266,
196
+ "step": 310
197
+ },
198
+ {
199
+ "grad_norm": 0.21465812623500824,
200
+ "learning_rate": 6.38e-05,
201
+ "loss": 0.0309,
202
+ "step": 320
203
+ },
204
+ {
205
+ "grad_norm": 0.22422099113464355,
206
+ "learning_rate": 6.58e-05,
207
+ "loss": 0.0345,
208
+ "step": 330
209
+ },
210
+ {
211
+ "grad_norm": 0.16468451917171478,
212
+ "learning_rate": 6.780000000000001e-05,
213
+ "loss": 0.0269,
214
+ "step": 340
215
+ },
216
+ {
217
+ "grad_norm": 0.1649431437253952,
218
+ "learning_rate": 6.98e-05,
219
+ "loss": 0.0285,
220
+ "step": 350
221
+ },
222
+ {
223
+ "grad_norm": 0.18796072900295258,
224
+ "learning_rate": 7.18e-05,
225
+ "loss": 0.0294,
226
+ "step": 360
227
+ },
228
+ {
229
+ "grad_norm": 0.1406984180212021,
230
+ "learning_rate": 7.38e-05,
231
+ "loss": 0.0262,
232
+ "step": 370
233
+ },
234
+ {
235
+ "grad_norm": 0.12663526833057404,
236
+ "learning_rate": 7.58e-05,
237
+ "loss": 0.0287,
238
+ "step": 380
239
+ },
240
+ {
241
+ "grad_norm": 0.17990556359291077,
242
+ "learning_rate": 7.780000000000001e-05,
243
+ "loss": 0.0358,
244
+ "step": 390
245
+ },
246
+ {
247
+ "grad_norm": 0.14493271708488464,
248
+ "learning_rate": 7.98e-05,
249
+ "loss": 0.0262,
250
+ "step": 400
251
+ },
252
+ {
253
+ "grad_norm": 0.18220767378807068,
254
+ "learning_rate": 8.18e-05,
255
+ "loss": 0.0307,
256
+ "step": 410
257
+ },
258
+ {
259
+ "grad_norm": 0.2229013890028,
260
+ "learning_rate": 8.38e-05,
261
+ "loss": 0.0393,
262
+ "step": 420
263
+ },
264
+ {
265
+ "grad_norm": 0.21163704991340637,
266
+ "learning_rate": 8.58e-05,
267
+ "loss": 0.0327,
268
+ "step": 430
269
+ },
270
+ {
271
+ "grad_norm": 0.1903618574142456,
272
+ "learning_rate": 8.78e-05,
273
+ "loss": 0.034,
274
+ "step": 440
275
+ },
276
+ {
277
+ "grad_norm": 0.19045576453208923,
278
+ "learning_rate": 8.98e-05,
279
+ "loss": 0.0334,
280
+ "step": 450
281
+ },
282
+ {
283
+ "grad_norm": 0.12919433414936066,
284
+ "learning_rate": 9.180000000000001e-05,
285
+ "loss": 0.0276,
286
+ "step": 460
287
+ },
288
+ {
289
+ "grad_norm": 0.22944214940071106,
290
+ "learning_rate": 9.38e-05,
291
+ "loss": 0.0366,
292
+ "step": 470
293
+ },
294
+ {
295
+ "grad_norm": 0.2297431230545044,
296
+ "learning_rate": 9.58e-05,
297
+ "loss": 0.031,
298
+ "step": 480
299
+ },
300
+ {
301
+ "grad_norm": 0.11911984533071518,
302
+ "learning_rate": 9.78e-05,
303
+ "loss": 0.0349,
304
+ "step": 490
305
+ },
306
+ {
307
+ "grad_norm": 0.14332818984985352,
308
+ "learning_rate": 9.98e-05,
309
+ "loss": 0.0298,
310
+ "step": 500
311
+ },
312
+ {
313
+ "grad_norm": 0.23667897284030914,
314
+ "learning_rate": 9.9999778549206e-05,
315
+ "loss": 0.0325,
316
+ "step": 510
317
+ },
318
+ {
319
+ "grad_norm": 0.1407025307416916,
320
+ "learning_rate": 9.999901304280685e-05,
321
+ "loss": 0.0275,
322
+ "step": 520
323
+ },
324
+ {
325
+ "grad_norm": 0.2208714336156845,
326
+ "learning_rate": 9.999770075521164e-05,
327
+ "loss": 0.0351,
328
+ "step": 530
329
+ },
330
+ {
331
+ "grad_norm": 0.18774442374706268,
332
+ "learning_rate": 9.99958417007713e-05,
333
+ "loss": 0.0415,
334
+ "step": 540
335
+ },
336
+ {
337
+ "grad_norm": 0.17202721536159515,
338
+ "learning_rate": 9.999343589981615e-05,
339
+ "loss": 0.0279,
340
+ "step": 550
341
+ },
342
+ {
343
+ "grad_norm": 0.1869555562734604,
344
+ "learning_rate": 9.999048337865568e-05,
345
+ "loss": 0.0325,
346
+ "step": 560
347
+ },
348
+ {
349
+ "grad_norm": 0.184967041015625,
350
+ "learning_rate": 9.998698416957815e-05,
351
+ "loss": 0.0426,
352
+ "step": 570
353
+ },
354
+ {
355
+ "grad_norm": 0.16213007271289825,
356
+ "learning_rate": 9.998293831085037e-05,
357
+ "loss": 0.0329,
358
+ "step": 580
359
+ },
360
+ {
361
+ "grad_norm": 0.1410742849111557,
362
+ "learning_rate": 9.997834584671719e-05,
363
+ "loss": 0.0313,
364
+ "step": 590
365
+ },
366
+ {
367
+ "grad_norm": 0.1963675171136856,
368
+ "learning_rate": 9.997320682740107e-05,
369
+ "loss": 0.0341,
370
+ "step": 600
371
+ },
372
+ {
373
+ "grad_norm": 0.18096579611301422,
374
+ "learning_rate": 9.996752130910149e-05,
375
+ "loss": 0.0378,
376
+ "step": 610
377
+ },
378
+ {
379
+ "grad_norm": 0.16188250482082367,
380
+ "learning_rate": 9.99612893539944e-05,
381
+ "loss": 0.0355,
382
+ "step": 620
383
+ },
384
+ {
385
+ "grad_norm": 0.20988130569458008,
386
+ "learning_rate": 9.995451103023144e-05,
387
+ "loss": 0.0327,
388
+ "step": 630
389
+ },
390
+ {
391
+ "grad_norm": 0.13830691576004028,
392
+ "learning_rate": 9.994718641193928e-05,
393
+ "loss": 0.0303,
394
+ "step": 640
395
+ },
396
+ {
397
+ "grad_norm": 0.10543777048587799,
398
+ "learning_rate": 9.993931557921874e-05,
399
+ "loss": 0.0311,
400
+ "step": 650
401
+ },
402
+ {
403
+ "grad_norm": 0.15627624094486237,
404
+ "learning_rate": 9.993089861814402e-05,
405
+ "loss": 0.0311,
406
+ "step": 660
407
+ },
408
+ {
409
+ "grad_norm": 0.14420993626117706,
410
+ "learning_rate": 9.992193562076166e-05,
411
+ "loss": 0.0256,
412
+ "step": 670
413
+ },
414
+ {
415
+ "grad_norm": 0.18450821936130524,
416
+ "learning_rate": 9.991242668508954e-05,
417
+ "loss": 0.04,
418
+ "step": 680
419
+ },
420
+ {
421
+ "grad_norm": 0.16379375755786896,
422
+ "learning_rate": 9.990237191511587e-05,
423
+ "loss": 0.0284,
424
+ "step": 690
425
+ },
426
+ {
427
+ "grad_norm": 0.14060579240322113,
428
+ "learning_rate": 9.989177142079802e-05,
429
+ "loss": 0.0276,
430
+ "step": 700
431
+ },
432
+ {
433
+ "grad_norm": 0.22451449930667877,
434
+ "learning_rate": 9.988062531806126e-05,
435
+ "loss": 0.0337,
436
+ "step": 710
437
+ },
438
+ {
439
+ "grad_norm": 0.13399986922740936,
440
+ "learning_rate": 9.986893372879762e-05,
441
+ "loss": 0.0292,
442
+ "step": 720
443
+ },
444
+ {
445
+ "grad_norm": 0.14917080104351044,
446
+ "learning_rate": 9.985669678086443e-05,
447
+ "loss": 0.0369,
448
+ "step": 730
449
+ },
450
+ {
451
+ "grad_norm": 0.2764376699924469,
452
+ "learning_rate": 9.984391460808298e-05,
453
+ "loss": 0.0334,
454
+ "step": 740
455
+ },
456
+ {
457
+ "grad_norm": 0.15942853689193726,
458
+ "learning_rate": 9.983058735023709e-05,
459
+ "loss": 0.0314,
460
+ "step": 750
461
+ },
462
+ {
463
+ "grad_norm": 0.19267232716083527,
464
+ "learning_rate": 9.98167151530715e-05,
465
+ "loss": 0.0302,
466
+ "step": 760
467
+ },
468
+ {
469
+ "grad_norm": 0.1857871413230896,
470
+ "learning_rate": 9.980229816829034e-05,
471
+ "loss": 0.0324,
472
+ "step": 770
473
+ },
474
+ {
475
+ "grad_norm": 0.09672777354717255,
476
+ "learning_rate": 9.978733655355544e-05,
477
+ "loss": 0.0336,
478
+ "step": 780
479
+ },
480
+ {
481
+ "grad_norm": 0.1687411516904831,
482
+ "learning_rate": 9.977183047248464e-05,
483
+ "loss": 0.0264,
484
+ "step": 790
485
+ },
486
+ {
487
+ "grad_norm": 0.15346349775791168,
488
+ "learning_rate": 9.975578009464992e-05,
489
+ "loss": 0.0342,
490
+ "step": 800
491
+ },
492
+ {
493
+ "grad_norm": 0.12507975101470947,
494
+ "learning_rate": 9.97391855955757e-05,
495
+ "loss": 0.0282,
496
+ "step": 810
497
+ },
498
+ {
499
+ "grad_norm": 0.1824805736541748,
500
+ "learning_rate": 9.972204715673669e-05,
501
+ "loss": 0.0475,
502
+ "step": 820
503
+ },
504
+ {
505
+ "grad_norm": 0.1841856837272644,
506
+ "learning_rate": 9.970436496555617e-05,
507
+ "loss": 0.0357,
508
+ "step": 830
509
+ },
510
+ {
511
+ "grad_norm": 0.19906769692897797,
512
+ "learning_rate": 9.968613921540373e-05,
513
+ "loss": 0.0295,
514
+ "step": 840
515
+ },
516
+ {
517
+ "grad_norm": 0.15382325649261475,
518
+ "learning_rate": 9.966737010559326e-05,
519
+ "loss": 0.0308,
520
+ "step": 850
521
+ },
522
+ {
523
+ "grad_norm": 0.16068829596042633,
524
+ "learning_rate": 9.964805784138072e-05,
525
+ "loss": 0.0353,
526
+ "step": 860
527
+ },
528
+ {
529
+ "grad_norm": 0.12692637741565704,
530
+ "learning_rate": 9.962820263396195e-05,
531
+ "loss": 0.0269,
532
+ "step": 870
533
+ },
534
+ {
535
+ "grad_norm": 0.17217664420604706,
536
+ "learning_rate": 9.960780470047033e-05,
537
+ "loss": 0.0329,
538
+ "step": 880
539
+ },
540
+ {
541
+ "grad_norm": 0.1513223648071289,
542
+ "learning_rate": 9.958686426397437e-05,
543
+ "loss": 0.0297,
544
+ "step": 890
545
+ },
546
+ {
547
+ "grad_norm": 0.14868152141571045,
548
+ "learning_rate": 9.956538155347534e-05,
549
+ "loss": 0.028,
550
+ "step": 900
551
+ },
552
+ {
553
+ "grad_norm": 0.16328009963035583,
554
+ "learning_rate": 9.95433568039047e-05,
555
+ "loss": 0.0308,
556
+ "step": 910
557
+ },
558
+ {
559
+ "grad_norm": 0.15237754583358765,
560
+ "learning_rate": 9.952079025612162e-05,
561
+ "loss": 0.0342,
562
+ "step": 920
563
+ },
564
+ {
565
+ "grad_norm": 0.17242884635925293,
566
+ "learning_rate": 9.949768215691022e-05,
567
+ "loss": 0.0404,
568
+ "step": 930
569
+ },
570
+ {
571
+ "grad_norm": 0.15023668110370636,
572
+ "learning_rate": 9.9474032758977e-05,
573
+ "loss": 0.0228,
574
+ "step": 940
575
+ },
576
+ {
577
+ "grad_norm": 0.16494718194007874,
578
+ "learning_rate": 9.944984232094794e-05,
579
+ "loss": 0.0287,
580
+ "step": 950
581
+ },
582
+ {
583
+ "grad_norm": 0.149717316031456,
584
+ "learning_rate": 9.942511110736584e-05,
585
+ "loss": 0.0337,
586
+ "step": 960
587
+ },
588
+ {
589
+ "grad_norm": 0.14683957397937775,
590
+ "learning_rate": 9.939983938868726e-05,
591
+ "loss": 0.0281,
592
+ "step": 970
593
+ },
594
+ {
595
+ "grad_norm": 0.18098339438438416,
596
+ "learning_rate": 9.93740274412797e-05,
597
+ "loss": 0.0309,
598
+ "step": 980
599
+ },
600
+ {
601
+ "grad_norm": 0.17116370797157288,
602
+ "learning_rate": 9.934767554741846e-05,
603
+ "loss": 0.0291,
604
+ "step": 990
605
+ },
606
+ {
607
+ "grad_norm": 0.13913299143314362,
608
+ "learning_rate": 9.932078399528361e-05,
609
+ "loss": 0.0267,
610
+ "step": 1000
611
+ },
612
+ {
613
+ "grad_norm": 0.1686856746673584,
614
+ "learning_rate": 9.929335307895689e-05,
615
+ "loss": 0.0363,
616
+ "step": 1010
617
+ },
618
+ {
619
+ "grad_norm": 0.13024087250232697,
620
+ "learning_rate": 9.926538309841839e-05,
621
+ "loss": 0.0285,
622
+ "step": 1020
623
+ },
624
+ {
625
+ "grad_norm": 0.17685119807720184,
626
+ "learning_rate": 9.923687435954334e-05,
627
+ "loss": 0.0287,
628
+ "step": 1030
629
+ },
630
+ {
631
+ "grad_norm": 0.20717529952526093,
632
+ "learning_rate": 9.920782717409873e-05,
633
+ "loss": 0.027,
634
+ "step": 1040
635
+ },
636
+ {
637
+ "grad_norm": 0.15997067093849182,
638
+ "learning_rate": 9.917824185973994e-05,
639
+ "loss": 0.0283,
640
+ "step": 1050
641
+ },
642
+ {
643
+ "grad_norm": 0.13489104807376862,
644
+ "learning_rate": 9.914811874000723e-05,
645
+ "loss": 0.0339,
646
+ "step": 1060
647
+ },
648
+ {
649
+ "grad_norm": 0.1213482990860939,
650
+ "learning_rate": 9.911745814432218e-05,
651
+ "loss": 0.0247,
652
+ "step": 1070
653
+ },
654
+ {
655
+ "grad_norm": 0.13703866302967072,
656
+ "learning_rate": 9.90862604079842e-05,
657
+ "loss": 0.0288,
658
+ "step": 1080
659
+ },
660
+ {
661
+ "grad_norm": 0.18348318338394165,
662
+ "learning_rate": 9.90545258721667e-05,
663
+ "loss": 0.0297,
664
+ "step": 1090
665
+ },
666
+ {
667
+ "grad_norm": 0.11433199793100357,
668
+ "learning_rate": 9.90222548839135e-05,
669
+ "loss": 0.0268,
670
+ "step": 1100
671
+ },
672
+ {
673
+ "grad_norm": 0.17286507785320282,
674
+ "learning_rate": 9.898944779613495e-05,
675
+ "loss": 0.0267,
676
+ "step": 1110
677
+ },
678
+ {
679
+ "grad_norm": 0.1405310034751892,
680
+ "learning_rate": 9.89561049676041e-05,
681
+ "loss": 0.0264,
682
+ "step": 1120
683
+ },
684
+ {
685
+ "grad_norm": 0.10237786918878555,
686
+ "learning_rate": 9.89222267629528e-05,
687
+ "loss": 0.0272,
688
+ "step": 1130
689
+ },
690
+ {
691
+ "grad_norm": 0.17079147696495056,
692
+ "learning_rate": 9.888781355266763e-05,
693
+ "loss": 0.0239,
694
+ "step": 1140
695
+ },
696
+ {
697
+ "grad_norm": 0.1960420310497284,
698
+ "learning_rate": 9.885286571308598e-05,
699
+ "loss": 0.0292,
700
+ "step": 1150
701
+ },
702
+ {
703
+ "grad_norm": 0.1341986507177353,
704
+ "learning_rate": 9.881738362639182e-05,
705
+ "loss": 0.0237,
706
+ "step": 1160
707
+ },
708
+ {
709
+ "grad_norm": 0.14957411587238312,
710
+ "learning_rate": 9.878136768061154e-05,
711
+ "loss": 0.0304,
712
+ "step": 1170
713
+ },
714
+ {
715
+ "grad_norm": 0.23255684971809387,
716
+ "learning_rate": 9.874481826960979e-05,
717
+ "loss": 0.0292,
718
+ "step": 1180
719
+ },
720
+ {
721
+ "grad_norm": 0.12505985796451569,
722
+ "learning_rate": 9.870773579308503e-05,
723
+ "loss": 0.0304,
724
+ "step": 1190
725
+ },
726
+ {
727
+ "grad_norm": 0.13537172973155975,
728
+ "learning_rate": 9.867012065656533e-05,
729
+ "loss": 0.0304,
730
+ "step": 1200
731
+ },
732
+ {
733
+ "grad_norm": 0.16207456588745117,
734
+ "learning_rate": 9.863197327140376e-05,
735
+ "loss": 0.035,
736
+ "step": 1210
737
+ },
738
+ {
739
+ "grad_norm": 0.22213716804981232,
740
+ "learning_rate": 9.859329405477403e-05,
741
+ "loss": 0.031,
742
+ "step": 1220
743
+ },
744
+ {
745
+ "grad_norm": 0.18120811879634857,
746
+ "learning_rate": 9.855408342966585e-05,
747
+ "loss": 0.0267,
748
+ "step": 1230
749
+ },
750
+ {
751
+ "grad_norm": 0.18635620176792145,
752
+ "learning_rate": 9.851434182488033e-05,
753
+ "loss": 0.0308,
754
+ "step": 1240
755
+ },
756
+ {
757
+ "grad_norm": 0.14335626363754272,
758
+ "learning_rate": 9.84740696750253e-05,
759
+ "loss": 0.0229,
760
+ "step": 1250
761
+ },
762
+ {
763
+ "grad_norm": 0.12551842629909515,
764
+ "learning_rate": 9.843326742051055e-05,
765
+ "loss": 0.0224,
766
+ "step": 1260
767
+ },
768
+ {
769
+ "grad_norm": 0.1251303106546402,
770
+ "learning_rate": 9.839193550754297e-05,
771
+ "loss": 0.0232,
772
+ "step": 1270
773
+ },
774
+ {
775
+ "grad_norm": 0.1565120369195938,
776
+ "learning_rate": 9.835007438812177e-05,
777
+ "loss": 0.03,
778
+ "step": 1280
779
+ },
780
+ {
781
+ "grad_norm": 0.15832260251045227,
782
+ "learning_rate": 9.830768452003341e-05,
783
+ "loss": 0.0271,
784
+ "step": 1290
785
+ },
786
+ {
787
+ "grad_norm": 0.13791042566299438,
788
+ "learning_rate": 9.826476636684671e-05,
789
+ "loss": 0.0276,
790
+ "step": 1300
791
+ },
792
+ {
793
+ "grad_norm": 0.10483084619045258,
794
+ "learning_rate": 9.822132039790773e-05,
795
+ "loss": 0.0248,
796
+ "step": 1310
797
+ },
798
+ {
799
+ "grad_norm": 0.14374341070652008,
800
+ "learning_rate": 9.817734708833461e-05,
801
+ "loss": 0.0269,
802
+ "step": 1320
803
+ },
804
+ {
805
+ "grad_norm": 0.16665680706501007,
806
+ "learning_rate": 9.813284691901243e-05,
807
+ "loss": 0.0307,
808
+ "step": 1330
809
+ },
810
+ {
811
+ "grad_norm": 0.21279016137123108,
812
+ "learning_rate": 9.808782037658792e-05,
813
+ "loss": 0.0333,
814
+ "step": 1340
815
+ },
816
+ {
817
+ "grad_norm": 0.1460651457309723,
818
+ "learning_rate": 9.804226795346411e-05,
819
+ "loss": 0.0242,
820
+ "step": 1350
821
+ },
822
+ {
823
+ "grad_norm": 0.20963597297668457,
824
+ "learning_rate": 9.799619014779503e-05,
825
+ "loss": 0.0321,
826
+ "step": 1360
827
+ },
828
+ {
829
+ "grad_norm": 0.19664771854877472,
830
+ "learning_rate": 9.794958746348013e-05,
831
+ "loss": 0.0257,
832
+ "step": 1370
833
+ },
834
+ {
835
+ "grad_norm": 0.2412680685520172,
836
+ "learning_rate": 9.790246041015896e-05,
837
+ "loss": 0.0265,
838
+ "step": 1380
839
+ },
840
+ {
841
+ "grad_norm": 0.2061948925256729,
842
+ "learning_rate": 9.785480950320538e-05,
843
+ "loss": 0.0289,
844
+ "step": 1390
845
+ },
846
+ {
847
+ "grad_norm": 0.1481916904449463,
848
+ "learning_rate": 9.78066352637221e-05,
849
+ "loss": 0.027,
850
+ "step": 1400
851
+ },
852
+ {
853
+ "grad_norm": 0.15597012639045715,
854
+ "learning_rate": 9.775793821853488e-05,
855
+ "loss": 0.0373,
856
+ "step": 1410
857
+ },
858
+ {
859
+ "grad_norm": 0.14834840595722198,
860
+ "learning_rate": 9.77087189001868e-05,
861
+ "loss": 0.0296,
862
+ "step": 1420
863
+ },
864
+ {
865
+ "grad_norm": 0.18620194494724274,
866
+ "learning_rate": 9.765897784693243e-05,
867
+ "loss": 0.0287,
868
+ "step": 1430
869
+ },
870
+ {
871
+ "grad_norm": 0.13414987921714783,
872
+ "learning_rate": 9.760871560273197e-05,
873
+ "loss": 0.0227,
874
+ "step": 1440
875
+ },
876
+ {
877
+ "grad_norm": 0.1416088491678238,
878
+ "learning_rate": 9.755793271724526e-05,
879
+ "loss": 0.0346,
880
+ "step": 1450
881
+ },
882
+ {
883
+ "grad_norm": 0.10656988620758057,
884
+ "learning_rate": 9.750662974582584e-05,
885
+ "loss": 0.027,
886
+ "step": 1460
887
+ },
888
+ {
889
+ "grad_norm": 0.12058360874652863,
890
+ "learning_rate": 9.745480724951473e-05,
891
+ "loss": 0.0199,
892
+ "step": 1470
893
+ },
894
+ {
895
+ "grad_norm": 0.21359531581401825,
896
+ "learning_rate": 9.740246579503447e-05,
897
+ "loss": 0.0251,
898
+ "step": 1480
899
+ },
900
+ {
901
+ "grad_norm": 0.15765275061130524,
902
+ "learning_rate": 9.734960595478284e-05,
903
+ "loss": 0.0245,
904
+ "step": 1490
905
+ },
906
+ {
907
+ "grad_norm": 0.13638831675052643,
908
+ "learning_rate": 9.729622830682657e-05,
909
+ "loss": 0.031,
910
+ "step": 1500
911
+ },
912
+ {
913
+ "grad_norm": 0.14469914138317108,
914
+ "learning_rate": 9.724233343489504e-05,
915
+ "loss": 0.0237,
916
+ "step": 1510
917
+ },
918
+ {
919
+ "grad_norm": 0.12858904898166656,
920
+ "learning_rate": 9.718792192837396e-05,
921
+ "loss": 0.028,
922
+ "step": 1520
923
+ },
924
+ {
925
+ "grad_norm": 0.18743933737277985,
926
+ "learning_rate": 9.713299438229886e-05,
927
+ "loss": 0.0266,
928
+ "step": 1530
929
+ },
930
+ {
931
+ "grad_norm": 0.17122581601142883,
932
+ "learning_rate": 9.707755139734855e-05,
933
+ "loss": 0.0249,
934
+ "step": 1540
935
+ },
936
+ {
937
+ "grad_norm": 0.125565305352211,
938
+ "learning_rate": 9.702159357983866e-05,
939
+ "loss": 0.0313,
940
+ "step": 1550
941
+ },
942
+ {
943
+ "grad_norm": 0.11599742621183395,
944
+ "learning_rate": 9.696512154171492e-05,
945
+ "loss": 0.0229,
946
+ "step": 1560
947
+ },
948
+ {
949
+ "grad_norm": 0.17274866998195648,
950
+ "learning_rate": 9.690813590054645e-05,
951
+ "loss": 0.0226,
952
+ "step": 1570
953
+ },
954
+ {
955
+ "grad_norm": 0.10541734844446182,
956
+ "learning_rate": 9.685063727951914e-05,
957
+ "loss": 0.0247,
958
+ "step": 1580
959
+ },
960
+ {
961
+ "grad_norm": 0.27482670545578003,
962
+ "learning_rate": 9.679262630742865e-05,
963
+ "loss": 0.0271,
964
+ "step": 1590
965
+ },
966
+ {
967
+ "grad_norm": 0.1570490002632141,
968
+ "learning_rate": 9.673410361867373e-05,
969
+ "loss": 0.0282,
970
+ "step": 1600
971
+ },
972
+ {
973
+ "grad_norm": 0.1765255182981491,
974
+ "learning_rate": 9.667506985324909e-05,
975
+ "loss": 0.0292,
976
+ "step": 1610
977
+ },
978
+ {
979
+ "grad_norm": 0.12382631003856659,
980
+ "learning_rate": 9.661552565673855e-05,
981
+ "loss": 0.0276,
982
+ "step": 1620
983
+ },
984
+ {
985
+ "grad_norm": 0.1488092988729477,
986
+ "learning_rate": 9.655547168030789e-05,
987
+ "loss": 0.0297,
988
+ "step": 1630
989
+ },
990
+ {
991
+ "grad_norm": 0.17621779441833496,
992
+ "learning_rate": 9.649490858069777e-05,
993
+ "loss": 0.0246,
994
+ "step": 1640
995
+ },
996
+ {
997
+ "grad_norm": 0.1317397505044937,
998
+ "learning_rate": 9.643383702021658e-05,
999
+ "loss": 0.0228,
1000
+ "step": 1650
1001
+ },
1002
+ {
1003
+ "grad_norm": 0.18702450394630432,
1004
+ "learning_rate": 9.637225766673307e-05,
1005
+ "loss": 0.0326,
1006
+ "step": 1660
1007
+ },
1008
+ {
1009
+ "grad_norm": 0.15152408182621002,
1010
+ "learning_rate": 9.631017119366922e-05,
1011
+ "loss": 0.025,
1012
+ "step": 1670
1013
+ },
1014
+ {
1015
+ "grad_norm": 0.14151324331760406,
1016
+ "learning_rate": 9.624757827999273e-05,
1017
+ "loss": 0.0234,
1018
+ "step": 1680
1019
+ },
1020
+ {
1021
+ "grad_norm": 0.17372380197048187,
1022
+ "learning_rate": 9.618447961020971e-05,
1023
+ "loss": 0.0361,
1024
+ "step": 1690
1025
+ },
1026
+ {
1027
+ "grad_norm": 0.14488667249679565,
1028
+ "learning_rate": 9.612087587435707e-05,
1029
+ "loss": 0.0248,
1030
+ "step": 1700
1031
+ },
1032
+ {
1033
+ "grad_norm": 0.1468273550271988,
1034
+ "learning_rate": 9.605676776799508e-05,
1035
+ "loss": 0.0236,
1036
+ "step": 1710
1037
+ },
1038
+ {
1039
+ "grad_norm": 0.16720327734947205,
1040
+ "learning_rate": 9.599215599219973e-05,
1041
+ "loss": 0.0299,
1042
+ "step": 1720
1043
+ },
1044
+ {
1045
+ "grad_norm": 0.10409308969974518,
1046
+ "learning_rate": 9.592704125355505e-05,
1047
+ "loss": 0.0224,
1048
+ "step": 1730
1049
+ },
1050
+ {
1051
+ "grad_norm": 0.22400495409965515,
1052
+ "learning_rate": 9.586142426414538e-05,
1053
+ "loss": 0.0298,
1054
+ "step": 1740
1055
+ },
1056
+ {
1057
+ "grad_norm": 0.16129139065742493,
1058
+ "learning_rate": 9.57953057415476e-05,
1059
+ "loss": 0.0216,
1060
+ "step": 1750
1061
+ },
1062
+ {
1063
+ "grad_norm": 0.15150457620620728,
1064
+ "learning_rate": 9.572868640882328e-05,
1065
+ "loss": 0.0255,
1066
+ "step": 1760
1067
+ },
1068
+ {
1069
+ "grad_norm": 0.27912425994873047,
1070
+ "learning_rate": 9.56615669945108e-05,
1071
+ "loss": 0.0265,
1072
+ "step": 1770
1073
+ },
1074
+ {
1075
+ "grad_norm": 0.15684591233730316,
1076
+ "learning_rate": 9.55939482326173e-05,
1077
+ "loss": 0.0231,
1078
+ "step": 1780
1079
+ },
1080
+ {
1081
+ "grad_norm": 0.1414581537246704,
1082
+ "learning_rate": 9.552583086261069e-05,
1083
+ "loss": 0.0298,
1084
+ "step": 1790
1085
+ },
1086
+ {
1087
+ "grad_norm": 0.12455716729164124,
1088
+ "learning_rate": 9.545721562941168e-05,
1089
+ "loss": 0.0311,
1090
+ "step": 1800
1091
+ },
1092
+ {
1093
+ "grad_norm": 0.1510583609342575,
1094
+ "learning_rate": 9.538810328338543e-05,
1095
+ "loss": 0.0245,
1096
+ "step": 1810
1097
+ },
1098
+ {
1099
+ "grad_norm": 0.12179003655910492,
1100
+ "learning_rate": 9.531849458033349e-05,
1101
+ "loss": 0.0195,
1102
+ "step": 1820
1103
+ },
1104
+ {
1105
+ "grad_norm": 0.15041644871234894,
1106
+ "learning_rate": 9.524839028148547e-05,
1107
+ "loss": 0.0237,
1108
+ "step": 1830
1109
+ },
1110
+ {
1111
+ "grad_norm": 0.18131820857524872,
1112
+ "learning_rate": 9.517779115349077e-05,
1113
+ "loss": 0.027,
1114
+ "step": 1840
1115
+ },
1116
+ {
1117
+ "grad_norm": 0.13325054943561554,
1118
+ "learning_rate": 9.510669796841014e-05,
1119
+ "loss": 0.0222,
1120
+ "step": 1850
1121
+ },
1122
+ {
1123
+ "grad_norm": 0.14594273269176483,
1124
+ "learning_rate": 9.503511150370727e-05,
1125
+ "loss": 0.0252,
1126
+ "step": 1860
1127
+ },
1128
+ {
1129
+ "grad_norm": 0.1101432591676712,
1130
+ "learning_rate": 9.496303254224024e-05,
1131
+ "loss": 0.0196,
1132
+ "step": 1870
1133
+ },
1134
+ {
1135
+ "grad_norm": 0.11899697780609131,
1136
+ "learning_rate": 9.489046187225306e-05,
1137
+ "loss": 0.0238,
1138
+ "step": 1880
1139
+ },
1140
+ {
1141
+ "grad_norm": 0.1834312528371811,
1142
+ "learning_rate": 9.481740028736692e-05,
1143
+ "loss": 0.027,
1144
+ "step": 1890
1145
+ },
1146
+ {
1147
+ "grad_norm": 0.1628856211900711,
1148
+ "learning_rate": 9.474384858657164e-05,
1149
+ "loss": 0.033,
1150
+ "step": 1900
1151
+ },
1152
+ {
1153
+ "grad_norm": 0.16760051250457764,
1154
+ "learning_rate": 9.466980757421679e-05,
1155
+ "loss": 0.0237,
1156
+ "step": 1910
1157
+ },
1158
+ {
1159
+ "grad_norm": 0.15349121391773224,
1160
+ "learning_rate": 9.459527806000305e-05,
1161
+ "loss": 0.0201,
1162
+ "step": 1920
1163
+ },
1164
+ {
1165
+ "grad_norm": 0.18761931359767914,
1166
+ "learning_rate": 9.452026085897325e-05,
1167
+ "loss": 0.0263,
1168
+ "step": 1930
1169
+ },
1170
+ {
1171
+ "grad_norm": 0.13588790595531464,
1172
+ "learning_rate": 9.444475679150348e-05,
1173
+ "loss": 0.0199,
1174
+ "step": 1940
1175
+ },
1176
+ {
1177
+ "grad_norm": 0.11293373256921768,
1178
+ "learning_rate": 9.436876668329411e-05,
1179
+ "loss": 0.0265,
1180
+ "step": 1950
1181
+ },
1182
+ {
1183
+ "grad_norm": 0.15826141834259033,
1184
+ "learning_rate": 9.429229136536079e-05,
1185
+ "loss": 0.0253,
1186
+ "step": 1960
1187
+ },
1188
+ {
1189
+ "grad_norm": 0.1520809531211853,
1190
+ "learning_rate": 9.421533167402534e-05,
1191
+ "loss": 0.0245,
1192
+ "step": 1970
1193
+ },
1194
+ {
1195
+ "grad_norm": 0.12347332388162613,
1196
+ "learning_rate": 9.413788845090666e-05,
1197
+ "loss": 0.0294,
1198
+ "step": 1980
1199
+ },
1200
+ {
1201
+ "grad_norm": 0.13870365917682648,
1202
+ "learning_rate": 9.405996254291136e-05,
1203
+ "loss": 0.022,
1204
+ "step": 1990
1205
+ },
1206
+ {
1207
+ "grad_norm": 0.1662704199552536,
1208
+ "learning_rate": 9.398155480222474e-05,
1209
+ "loss": 0.0267,
1210
+ "step": 2000
1211
+ },
1212
+ {
1213
+ "grad_norm": 0.13557152450084686,
1214
+ "learning_rate": 9.390266608630128e-05,
1215
+ "loss": 0.0227,
1216
+ "step": 2010
1217
+ },
1218
+ {
1219
+ "grad_norm": 0.0978504940867424,
1220
+ "learning_rate": 9.38232972578553e-05,
1221
+ "loss": 0.0228,
1222
+ "step": 2020
1223
+ },
1224
+ {
1225
+ "grad_norm": 0.1562250852584839,
1226
+ "learning_rate": 9.374344918485164e-05,
1227
+ "loss": 0.022,
1228
+ "step": 2030
1229
+ },
1230
+ {
1231
+ "grad_norm": 0.12346863746643066,
1232
+ "learning_rate": 9.366312274049602e-05,
1233
+ "loss": 0.0227,
1234
+ "step": 2040
1235
+ },
1236
+ {
1237
+ "grad_norm": 0.12018659710884094,
1238
+ "learning_rate": 9.358231880322554e-05,
1239
+ "loss": 0.0217,
1240
+ "step": 2050
1241
+ },
1242
+ {
1243
+ "grad_norm": 0.1706809550523758,
1244
+ "learning_rate": 9.350103825669916e-05,
1245
+ "loss": 0.025,
1246
+ "step": 2060
1247
+ },
1248
+ {
1249
+ "grad_norm": 0.1152491346001625,
1250
+ "learning_rate": 9.341928198978787e-05,
1251
+ "loss": 0.0226,
1252
+ "step": 2070
1253
+ },
1254
+ {
1255
+ "grad_norm": 0.1519344598054886,
1256
+ "learning_rate": 9.333705089656512e-05,
1257
+ "loss": 0.0242,
1258
+ "step": 2080
1259
+ },
1260
+ {
1261
+ "grad_norm": 0.16569732129573822,
1262
+ "learning_rate": 9.325434587629698e-05,
1263
+ "loss": 0.0196,
1264
+ "step": 2090
1265
+ },
1266
+ {
1267
+ "grad_norm": 0.16990657150745392,
1268
+ "learning_rate": 9.31711678334323e-05,
1269
+ "loss": 0.022,
1270
+ "step": 2100
1271
+ },
1272
+ {
1273
+ "grad_norm": 0.16231462359428406,
1274
+ "learning_rate": 9.308751767759282e-05,
1275
+ "loss": 0.0248,
1276
+ "step": 2110
1277
+ },
1278
+ {
1279
+ "grad_norm": 0.2376696765422821,
1280
+ "learning_rate": 9.300339632356325e-05,
1281
+ "loss": 0.0329,
1282
+ "step": 2120
1283
+ },
1284
+ {
1285
+ "grad_norm": 0.2505277395248413,
1286
+ "learning_rate": 9.291880469128124e-05,
1287
+ "loss": 0.0299,
1288
+ "step": 2130
1289
+ },
1290
+ {
1291
+ "grad_norm": 0.196861132979393,
1292
+ "learning_rate": 9.283374370582732e-05,
1293
+ "loss": 0.0239,
1294
+ "step": 2140
1295
+ },
1296
+ {
1297
+ "grad_norm": 0.13766983151435852,
1298
+ "learning_rate": 9.274821429741482e-05,
1299
+ "loss": 0.0217,
1300
+ "step": 2150
1301
+ },
1302
+ {
1303
+ "grad_norm": 0.1426539272069931,
1304
+ "learning_rate": 9.266221740137961e-05,
1305
+ "loss": 0.0309,
1306
+ "step": 2160
1307
+ },
1308
+ {
1309
+ "grad_norm": 0.14772582054138184,
1310
+ "learning_rate": 9.257575395817001e-05,
1311
+ "loss": 0.0192,
1312
+ "step": 2170
1313
+ },
1314
+ {
1315
+ "grad_norm": 0.12440606206655502,
1316
+ "learning_rate": 9.248882491333637e-05,
1317
+ "loss": 0.0172,
1318
+ "step": 2180
1319
+ },
1320
+ {
1321
+ "grad_norm": 0.17530487477779388,
1322
+ "learning_rate": 9.240143121752076e-05,
1323
+ "loss": 0.0234,
1324
+ "step": 2190
1325
+ },
1326
+ {
1327
+ "grad_norm": 0.12355060875415802,
1328
+ "learning_rate": 9.23135738264467e-05,
1329
+ "loss": 0.0218,
1330
+ "step": 2200
1331
+ },
1332
+ {
1333
+ "grad_norm": 0.21175123751163483,
1334
+ "learning_rate": 9.222525370090849e-05,
1335
+ "loss": 0.0274,
1336
+ "step": 2210
1337
+ },
1338
+ {
1339
+ "grad_norm": 0.14347153902053833,
1340
+ "learning_rate": 9.213647180676088e-05,
1341
+ "loss": 0.0238,
1342
+ "step": 2220
1343
+ },
1344
+ {
1345
+ "grad_norm": 0.11950457096099854,
1346
+ "learning_rate": 9.204722911490846e-05,
1347
+ "loss": 0.0278,
1348
+ "step": 2230
1349
+ },
1350
+ {
1351
+ "grad_norm": 0.14002735912799835,
1352
+ "learning_rate": 9.1957526601295e-05,
1353
+ "loss": 0.0235,
1354
+ "step": 2240
1355
+ },
1356
+ {
1357
+ "grad_norm": 0.13814957439899445,
1358
+ "learning_rate": 9.186736524689281e-05,
1359
+ "loss": 0.0243,
1360
+ "step": 2250
1361
+ },
1362
+ {
1363
+ "grad_norm": 0.13607311248779297,
1364
+ "learning_rate": 9.177674603769204e-05,
1365
+ "loss": 0.0341,
1366
+ "step": 2260
1367
+ },
1368
+ {
1369
+ "grad_norm": 0.16439314186573029,
1370
+ "learning_rate": 9.168566996468983e-05,
1371
+ "loss": 0.0266,
1372
+ "step": 2270
1373
+ },
1374
+ {
1375
+ "grad_norm": 0.1378590613603592,
1376
+ "learning_rate": 9.159413802387951e-05,
1377
+ "loss": 0.0258,
1378
+ "step": 2280
1379
+ },
1380
+ {
1381
+ "grad_norm": 0.18438801169395447,
1382
+ "learning_rate": 9.150215121623974e-05,
1383
+ "loss": 0.027,
1384
+ "step": 2290
1385
+ },
1386
+ {
1387
+ "grad_norm": 0.12811657786369324,
1388
+ "learning_rate": 9.140971054772349e-05,
1389
+ "loss": 0.0233,
1390
+ "step": 2300
1391
+ },
1392
+ {
1393
+ "grad_norm": 0.14623720943927765,
1394
+ "learning_rate": 9.131681702924713e-05,
1395
+ "loss": 0.0204,
1396
+ "step": 2310
1397
+ },
1398
+ {
1399
+ "grad_norm": 0.12561063468456268,
1400
+ "learning_rate": 9.122347167667926e-05,
1401
+ "loss": 0.0263,
1402
+ "step": 2320
1403
+ },
1404
+ {
1405
+ "grad_norm": 0.17007383704185486,
1406
+ "learning_rate": 9.112967551082973e-05,
1407
+ "loss": 0.0234,
1408
+ "step": 2330
1409
+ },
1410
+ {
1411
+ "grad_norm": 0.11141370236873627,
1412
+ "learning_rate": 9.103542955743835e-05,
1413
+ "loss": 0.0248,
1414
+ "step": 2340
1415
+ },
1416
+ {
1417
+ "grad_norm": 0.18721242249011993,
1418
+ "learning_rate": 9.094073484716381e-05,
1419
+ "loss": 0.0273,
1420
+ "step": 2350
1421
+ },
1422
+ {
1423
+ "grad_norm": 0.14813850820064545,
1424
+ "learning_rate": 9.084559241557226e-05,
1425
+ "loss": 0.0212,
1426
+ "step": 2360
1427
+ },
1428
+ {
1429
+ "grad_norm": 0.15478885173797607,
1430
+ "learning_rate": 9.075000330312608e-05,
1431
+ "loss": 0.0269,
1432
+ "step": 2370
1433
+ },
1434
+ {
1435
+ "grad_norm": 0.15512995421886444,
1436
+ "learning_rate": 9.065396855517253e-05,
1437
+ "loss": 0.0241,
1438
+ "step": 2380
1439
+ },
1440
+ {
1441
+ "grad_norm": 0.18194761872291565,
1442
+ "learning_rate": 9.055748922193219e-05,
1443
+ "loss": 0.0298,
1444
+ "step": 2390
1445
+ },
1446
+ {
1447
+ "grad_norm": 0.09133775532245636,
1448
+ "learning_rate": 9.046056635848761e-05,
1449
+ "loss": 0.0202,
1450
+ "step": 2400
1451
+ },
1452
+ {
1453
+ "grad_norm": 0.14668187499046326,
1454
+ "learning_rate": 9.036320102477169e-05,
1455
+ "loss": 0.0235,
1456
+ "step": 2410
1457
+ },
1458
+ {
1459
+ "grad_norm": 0.1325305700302124,
1460
+ "learning_rate": 9.02653942855561e-05,
1461
+ "loss": 0.0245,
1462
+ "step": 2420
1463
+ },
1464
+ {
1465
+ "grad_norm": 0.14857754111289978,
1466
+ "learning_rate": 9.016714721043971e-05,
1467
+ "loss": 0.0205,
1468
+ "step": 2430
1469
+ },
1470
+ {
1471
+ "grad_norm": 0.16199085116386414,
1472
+ "learning_rate": 9.006846087383675e-05,
1473
+ "loss": 0.0213,
1474
+ "step": 2440
1475
+ },
1476
+ {
1477
+ "grad_norm": 0.10761445760726929,
1478
+ "learning_rate": 8.996933635496523e-05,
1479
+ "loss": 0.0278,
1480
+ "step": 2450
1481
+ },
1482
+ {
1483
+ "grad_norm": 0.1585974097251892,
1484
+ "learning_rate": 8.986977473783498e-05,
1485
+ "loss": 0.0241,
1486
+ "step": 2460
1487
+ },
1488
+ {
1489
+ "grad_norm": 0.10342201590538025,
1490
+ "learning_rate": 8.97697771112359e-05,
1491
+ "loss": 0.0212,
1492
+ "step": 2470
1493
+ },
1494
+ {
1495
+ "grad_norm": 0.14516714215278625,
1496
+ "learning_rate": 8.966934456872602e-05,
1497
+ "loss": 0.0243,
1498
+ "step": 2480
1499
+ },
1500
+ {
1501
+ "grad_norm": 0.14087602496147156,
1502
+ "learning_rate": 8.95684782086195e-05,
1503
+ "loss": 0.0279,
1504
+ "step": 2490
1505
+ },
1506
+ {
1507
+ "grad_norm": 0.12304757535457611,
1508
+ "learning_rate": 8.946717913397476e-05,
1509
+ "loss": 0.027,
1510
+ "step": 2500
1511
+ },
1512
+ {
1513
+ "grad_norm": 0.1442318856716156,
1514
+ "learning_rate": 8.93654484525822e-05,
1515
+ "loss": 0.0239,
1516
+ "step": 2510
1517
+ },
1518
+ {
1519
+ "grad_norm": 0.15643472969532013,
1520
+ "learning_rate": 8.926328727695226e-05,
1521
+ "loss": 0.0205,
1522
+ "step": 2520
1523
+ },
1524
+ {
1525
+ "grad_norm": 0.09812328964471817,
1526
+ "learning_rate": 8.916069672430319e-05,
1527
+ "loss": 0.0214,
1528
+ "step": 2530
1529
+ },
1530
+ {
1531
+ "grad_norm": 0.1295197308063507,
1532
+ "learning_rate": 8.905767791654884e-05,
1533
+ "loss": 0.0302,
1534
+ "step": 2540
1535
+ },
1536
+ {
1537
+ "grad_norm": 0.13791200518608093,
1538
+ "learning_rate": 8.895423198028638e-05,
1539
+ "loss": 0.0199,
1540
+ "step": 2550
1541
+ },
1542
+ {
1543
+ "grad_norm": 0.16387410461902618,
1544
+ "learning_rate": 8.885036004678402e-05,
1545
+ "loss": 0.0289,
1546
+ "step": 2560
1547
+ },
1548
+ {
1549
+ "grad_norm": 0.15156884491443634,
1550
+ "learning_rate": 8.874606325196857e-05,
1551
+ "loss": 0.0265,
1552
+ "step": 2570
1553
+ },
1554
+ {
1555
+ "grad_norm": 0.12447788566350937,
1556
+ "learning_rate": 8.864134273641304e-05,
1557
+ "loss": 0.0243,
1558
+ "step": 2580
1559
+ },
1560
+ {
1561
+ "grad_norm": 0.11832483112812042,
1562
+ "learning_rate": 8.853619964532427e-05,
1563
+ "loss": 0.0219,
1564
+ "step": 2590
1565
+ },
1566
+ {
1567
+ "grad_norm": 0.15960167348384857,
1568
+ "learning_rate": 8.843063512853019e-05,
1569
+ "loss": 0.0262,
1570
+ "step": 2600
1571
+ },
1572
+ {
1573
+ "grad_norm": 0.16603811085224152,
1574
+ "learning_rate": 8.832465034046749e-05,
1575
+ "loss": 0.0228,
1576
+ "step": 2610
1577
+ },
1578
+ {
1579
+ "grad_norm": 0.1218847781419754,
1580
+ "learning_rate": 8.821824644016882e-05,
1581
+ "loss": 0.0205,
1582
+ "step": 2620
1583
+ },
1584
+ {
1585
+ "grad_norm": 0.1599685251712799,
1586
+ "learning_rate": 8.811142459125019e-05,
1587
+ "loss": 0.0259,
1588
+ "step": 2630
1589
+ },
1590
+ {
1591
+ "grad_norm": 0.10958845168352127,
1592
+ "learning_rate": 8.800418596189822e-05,
1593
+ "loss": 0.0236,
1594
+ "step": 2640
1595
+ },
1596
+ {
1597
+ "grad_norm": 0.12428676337003708,
1598
+ "learning_rate": 8.789653172485737e-05,
1599
+ "loss": 0.0208,
1600
+ "step": 2650
1601
+ },
1602
+ {
1603
+ "grad_norm": 0.12821273505687714,
1604
+ "learning_rate": 8.778846305741715e-05,
1605
+ "loss": 0.0205,
1606
+ "step": 2660
1607
+ },
1608
+ {
1609
+ "grad_norm": 0.11910676211118698,
1610
+ "learning_rate": 8.767998114139918e-05,
1611
+ "loss": 0.0261,
1612
+ "step": 2670
1613
+ },
1614
+ {
1615
+ "grad_norm": 0.1570224165916443,
1616
+ "learning_rate": 8.757108716314429e-05,
1617
+ "loss": 0.0206,
1618
+ "step": 2680
1619
+ },
1620
+ {
1621
+ "grad_norm": 0.1630830466747284,
1622
+ "learning_rate": 8.746178231349962e-05,
1623
+ "loss": 0.0203,
1624
+ "step": 2690
1625
+ },
1626
+ {
1627
+ "grad_norm": 0.07918090373277664,
1628
+ "learning_rate": 8.735206778780549e-05,
1629
+ "loss": 0.0222,
1630
+ "step": 2700
1631
+ },
1632
+ {
1633
+ "grad_norm": 0.1236148476600647,
1634
+ "learning_rate": 8.724194478588234e-05,
1635
+ "loss": 0.0194,
1636
+ "step": 2710
1637
+ },
1638
+ {
1639
+ "grad_norm": 0.15769383311271667,
1640
+ "learning_rate": 8.713141451201772e-05,
1641
+ "loss": 0.0252,
1642
+ "step": 2720
1643
+ },
1644
+ {
1645
+ "grad_norm": 0.10508900135755539,
1646
+ "learning_rate": 8.702047817495295e-05,
1647
+ "loss": 0.0171,
1648
+ "step": 2730
1649
+ },
1650
+ {
1651
+ "grad_norm": 0.12346747517585754,
1652
+ "learning_rate": 8.69091369878701e-05,
1653
+ "loss": 0.0191,
1654
+ "step": 2740
1655
+ },
1656
+ {
1657
+ "grad_norm": 0.1627315729856491,
1658
+ "learning_rate": 8.679739216837849e-05,
1659
+ "loss": 0.0228,
1660
+ "step": 2750
1661
+ },
1662
+ {
1663
+ "grad_norm": 0.111842580139637,
1664
+ "learning_rate": 8.66852449385016e-05,
1665
+ "loss": 0.021,
1666
+ "step": 2760
1667
+ },
1668
+ {
1669
+ "grad_norm": 0.111451655626297,
1670
+ "learning_rate": 8.657269652466356e-05,
1671
+ "loss": 0.0188,
1672
+ "step": 2770
1673
+ },
1674
+ {
1675
+ "grad_norm": 0.12247065454721451,
1676
+ "learning_rate": 8.645974815767577e-05,
1677
+ "loss": 0.02,
1678
+ "step": 2780
1679
+ },
1680
+ {
1681
+ "grad_norm": 0.18082000315189362,
1682
+ "learning_rate": 8.634640107272351e-05,
1683
+ "loss": 0.0254,
1684
+ "step": 2790
1685
+ },
1686
+ {
1687
+ "grad_norm": 0.14393867552280426,
1688
+ "learning_rate": 8.623265650935234e-05,
1689
+ "loss": 0.0231,
1690
+ "step": 2800
1691
+ },
1692
+ {
1693
+ "grad_norm": 0.14712563157081604,
1694
+ "learning_rate": 8.611851571145456e-05,
1695
+ "loss": 0.0189,
1696
+ "step": 2810
1697
+ },
1698
+ {
1699
+ "grad_norm": 0.11501806229352951,
1700
+ "learning_rate": 8.600397992725566e-05,
1701
+ "loss": 0.0179,
1702
+ "step": 2820
1703
+ },
1704
+ {
1705
+ "grad_norm": 0.14018242061138153,
1706
+ "learning_rate": 8.588905040930061e-05,
1707
+ "loss": 0.0275,
1708
+ "step": 2830
1709
+ },
1710
+ {
1711
+ "grad_norm": 0.15879635512828827,
1712
+ "learning_rate": 8.577372841444022e-05,
1713
+ "loss": 0.0213,
1714
+ "step": 2840
1715
+ },
1716
+ {
1717
+ "grad_norm": 0.14423353970050812,
1718
+ "learning_rate": 8.565801520381736e-05,
1719
+ "loss": 0.0209,
1720
+ "step": 2850
1721
+ },
1722
+ {
1723
+ "grad_norm": 0.13850180804729462,
1724
+ "learning_rate": 8.554191204285313e-05,
1725
+ "loss": 0.0211,
1726
+ "step": 2860
1727
+ },
1728
+ {
1729
+ "grad_norm": 0.13924437761306763,
1730
+ "learning_rate": 8.542542020123315e-05,
1731
+ "loss": 0.0238,
1732
+ "step": 2870
1733
+ },
1734
+ {
1735
+ "grad_norm": 0.09380088746547699,
1736
+ "learning_rate": 8.530854095289347e-05,
1737
+ "loss": 0.0186,
1738
+ "step": 2880
1739
+ },
1740
+ {
1741
+ "grad_norm": 0.1399223804473877,
1742
+ "learning_rate": 8.519127557600688e-05,
1743
+ "loss": 0.0247,
1744
+ "step": 2890
1745
+ },
1746
+ {
1747
+ "grad_norm": 0.13163888454437256,
1748
+ "learning_rate": 8.507362535296871e-05,
1749
+ "loss": 0.0215,
1750
+ "step": 2900
1751
+ },
1752
+ {
1753
+ "grad_norm": 0.12561990320682526,
1754
+ "learning_rate": 8.495559157038299e-05,
1755
+ "loss": 0.0194,
1756
+ "step": 2910
1757
+ },
1758
+ {
1759
+ "grad_norm": 0.12927483022212982,
1760
+ "learning_rate": 8.483717551904823e-05,
1761
+ "loss": 0.0221,
1762
+ "step": 2920
1763
+ },
1764
+ {
1765
+ "grad_norm": 0.12346947938203812,
1766
+ "learning_rate": 8.47183784939434e-05,
1767
+ "loss": 0.0203,
1768
+ "step": 2930
1769
+ },
1770
+ {
1771
+ "grad_norm": 0.13767321407794952,
1772
+ "learning_rate": 8.459920179421374e-05,
1773
+ "loss": 0.0247,
1774
+ "step": 2940
1775
+ },
1776
+ {
1777
+ "grad_norm": 0.17473234236240387,
1778
+ "learning_rate": 8.447964672315656e-05,
1779
+ "loss": 0.0277,
1780
+ "step": 2950
1781
+ },
1782
+ {
1783
+ "grad_norm": 0.13432490825653076,
1784
+ "learning_rate": 8.435971458820692e-05,
1785
+ "loss": 0.0234,
1786
+ "step": 2960
1787
+ },
1788
+ {
1789
+ "grad_norm": 0.1274847835302353,
1790
+ "learning_rate": 8.423940670092345e-05,
1791
+ "loss": 0.0212,
1792
+ "step": 2970
1793
+ },
1794
+ {
1795
+ "grad_norm": 0.0896708145737648,
1796
+ "learning_rate": 8.411872437697394e-05,
1797
+ "loss": 0.0188,
1798
+ "step": 2980
1799
+ },
1800
+ {
1801
+ "grad_norm": 0.13214024901390076,
1802
+ "learning_rate": 8.399766893612096e-05,
1803
+ "loss": 0.0231,
1804
+ "step": 2990
1805
+ },
1806
+ {
1807
+ "grad_norm": 0.13123571872711182,
1808
+ "learning_rate": 8.38762417022074e-05,
1809
+ "loss": 0.0316,
1810
+ "step": 3000
1811
+ },
1812
+ {
1813
+ "grad_norm": 0.13476744294166565,
1814
+ "learning_rate": 8.375444400314204e-05,
1815
+ "loss": 0.0203,
1816
+ "step": 3010
1817
+ },
1818
+ {
1819
+ "grad_norm": 0.12753205001354218,
1820
+ "learning_rate": 8.3632277170885e-05,
1821
+ "loss": 0.0222,
1822
+ "step": 3020
1823
+ },
1824
+ {
1825
+ "grad_norm": 0.10575438290834427,
1826
+ "learning_rate": 8.350974254143318e-05,
1827
+ "loss": 0.021,
1828
+ "step": 3030
1829
+ },
1830
+ {
1831
+ "grad_norm": 0.1535600870847702,
1832
+ "learning_rate": 8.338684145480566e-05,
1833
+ "loss": 0.0215,
1834
+ "step": 3040
1835
+ },
1836
+ {
1837
+ "grad_norm": 0.21555498242378235,
1838
+ "learning_rate": 8.326357525502904e-05,
1839
+ "loss": 0.0271,
1840
+ "step": 3050
1841
+ },
1842
+ {
1843
+ "grad_norm": 0.14917391538619995,
1844
+ "learning_rate": 8.313994529012273e-05,
1845
+ "loss": 0.0196,
1846
+ "step": 3060
1847
+ },
1848
+ {
1849
+ "grad_norm": 0.10461319237947464,
1850
+ "learning_rate": 8.301595291208422e-05,
1851
+ "loss": 0.0185,
1852
+ "step": 3070
1853
+ },
1854
+ {
1855
+ "grad_norm": 0.21023470163345337,
1856
+ "learning_rate": 8.289159947687427e-05,
1857
+ "loss": 0.0355,
1858
+ "step": 3080
1859
+ },
1860
+ {
1861
+ "grad_norm": 0.15654537081718445,
1862
+ "learning_rate": 8.276688634440216e-05,
1863
+ "loss": 0.0245,
1864
+ "step": 3090
1865
+ },
1866
+ {
1867
+ "grad_norm": 0.1336844116449356,
1868
+ "learning_rate": 8.26418148785107e-05,
1869
+ "loss": 0.0261,
1870
+ "step": 3100
1871
+ },
1872
+ {
1873
+ "grad_norm": 0.15251711010932922,
1874
+ "learning_rate": 8.251638644696141e-05,
1875
+ "loss": 0.021,
1876
+ "step": 3110
1877
+ },
1878
+ {
1879
+ "grad_norm": 0.1364535242319107,
1880
+ "learning_rate": 8.23906024214195e-05,
1881
+ "loss": 0.0233,
1882
+ "step": 3120
1883
+ },
1884
+ {
1885
+ "grad_norm": 0.14940956234931946,
1886
+ "learning_rate": 8.226446417743897e-05,
1887
+ "loss": 0.0187,
1888
+ "step": 3130
1889
+ },
1890
+ {
1891
+ "grad_norm": 0.12878847122192383,
1892
+ "learning_rate": 8.213797309444742e-05,
1893
+ "loss": 0.0186,
1894
+ "step": 3140
1895
+ },
1896
+ {
1897
+ "grad_norm": 0.1497132033109665,
1898
+ "learning_rate": 8.201113055573105e-05,
1899
+ "loss": 0.02,
1900
+ "step": 3150
1901
+ },
1902
+ {
1903
+ "grad_norm": 0.1366586536169052,
1904
+ "learning_rate": 8.188393794841958e-05,
1905
+ "loss": 0.0172,
1906
+ "step": 3160
1907
+ },
1908
+ {
1909
+ "grad_norm": 0.13715018332004547,
1910
+ "learning_rate": 8.175639666347094e-05,
1911
+ "loss": 0.0258,
1912
+ "step": 3170
1913
+ },
1914
+ {
1915
+ "grad_norm": 0.15244945883750916,
1916
+ "learning_rate": 8.162850809565623e-05,
1917
+ "loss": 0.023,
1918
+ "step": 3180
1919
+ },
1920
+ {
1921
+ "grad_norm": 0.20791588723659515,
1922
+ "learning_rate": 8.150027364354431e-05,
1923
+ "loss": 0.0182,
1924
+ "step": 3190
1925
+ },
1926
+ {
1927
+ "grad_norm": 0.11517085880041122,
1928
+ "learning_rate": 8.137169470948662e-05,
1929
+ "loss": 0.0238,
1930
+ "step": 3200
1931
+ },
1932
+ {
1933
+ "grad_norm": 0.1119120642542839,
1934
+ "learning_rate": 8.124277269960179e-05,
1935
+ "loss": 0.0192,
1936
+ "step": 3210
1937
+ },
1938
+ {
1939
+ "grad_norm": 0.10967746376991272,
1940
+ "learning_rate": 8.111350902376023e-05,
1941
+ "loss": 0.0187,
1942
+ "step": 3220
1943
+ },
1944
+ {
1945
+ "grad_norm": 0.1142214760184288,
1946
+ "learning_rate": 8.098390509556883e-05,
1947
+ "loss": 0.0183,
1948
+ "step": 3230
1949
+ },
1950
+ {
1951
+ "grad_norm": 0.12059076130390167,
1952
+ "learning_rate": 8.085396233235536e-05,
1953
+ "loss": 0.0164,
1954
+ "step": 3240
1955
+ },
1956
+ {
1957
+ "grad_norm": 0.10252851992845535,
1958
+ "learning_rate": 8.072368215515306e-05,
1959
+ "loss": 0.0198,
1960
+ "step": 3250
1961
+ },
1962
+ {
1963
+ "grad_norm": 0.13169099390506744,
1964
+ "learning_rate": 8.059306598868506e-05,
1965
+ "loss": 0.0166,
1966
+ "step": 3260
1967
+ },
1968
+ {
1969
+ "grad_norm": 0.12112400680780411,
1970
+ "learning_rate": 8.046211526134888e-05,
1971
+ "loss": 0.0206,
1972
+ "step": 3270
1973
+ },
1974
+ {
1975
+ "grad_norm": 0.10669714212417603,
1976
+ "learning_rate": 8.033083140520065e-05,
1977
+ "loss": 0.0197,
1978
+ "step": 3280
1979
+ },
1980
+ {
1981
+ "grad_norm": 0.19681450724601746,
1982
+ "learning_rate": 8.019921585593962e-05,
1983
+ "loss": 0.0202,
1984
+ "step": 3290
1985
+ },
1986
+ {
1987
+ "grad_norm": 0.1599496304988861,
1988
+ "learning_rate": 8.006727005289232e-05,
1989
+ "loss": 0.0282,
1990
+ "step": 3300
1991
+ },
1992
+ {
1993
+ "grad_norm": 0.14205233752727509,
1994
+ "learning_rate": 7.993499543899692e-05,
1995
+ "loss": 0.017,
1996
+ "step": 3310
1997
+ },
1998
+ {
1999
+ "grad_norm": 0.14852288365364075,
2000
+ "learning_rate": 7.980239346078742e-05,
2001
+ "loss": 0.0176,
2002
+ "step": 3320
2003
+ },
2004
+ {
2005
+ "grad_norm": 0.12392748147249222,
2006
+ "learning_rate": 7.966946556837778e-05,
2007
+ "loss": 0.0204,
2008
+ "step": 3330
2009
+ },
2010
+ {
2011
+ "grad_norm": 0.12351145595312119,
2012
+ "learning_rate": 7.953621321544616e-05,
2013
+ "loss": 0.0196,
2014
+ "step": 3340
2015
+ },
2016
+ {
2017
+ "grad_norm": 0.13077925145626068,
2018
+ "learning_rate": 7.940263785921896e-05,
2019
+ "loss": 0.0199,
2020
+ "step": 3350
2021
+ },
2022
+ {
2023
+ "grad_norm": 0.09509874135255814,
2024
+ "learning_rate": 7.926874096045482e-05,
2025
+ "loss": 0.0235,
2026
+ "step": 3360
2027
+ },
2028
+ {
2029
+ "grad_norm": 0.12468800693750381,
2030
+ "learning_rate": 7.913452398342881e-05,
2031
+ "loss": 0.0271,
2032
+ "step": 3370
2033
+ },
2034
+ {
2035
+ "grad_norm": 0.09213206171989441,
2036
+ "learning_rate": 7.89999883959163e-05,
2037
+ "loss": 0.0179,
2038
+ "step": 3380
2039
+ },
2040
+ {
2041
+ "grad_norm": 0.13299454748630524,
2042
+ "learning_rate": 7.886513566917687e-05,
2043
+ "loss": 0.0193,
2044
+ "step": 3390
2045
+ },
2046
+ {
2047
+ "grad_norm": 0.13831903040409088,
2048
+ "learning_rate": 7.872996727793838e-05,
2049
+ "loss": 0.0199,
2050
+ "step": 3400
2051
+ },
2052
+ {
2053
+ "grad_norm": 0.14821100234985352,
2054
+ "learning_rate": 7.859448470038069e-05,
2055
+ "loss": 0.0177,
2056
+ "step": 3410
2057
+ },
2058
+ {
2059
+ "grad_norm": 0.1522444635629654,
2060
+ "learning_rate": 7.845868941811956e-05,
2061
+ "loss": 0.0201,
2062
+ "step": 3420
2063
+ },
2064
+ {
2065
+ "grad_norm": 0.13491684198379517,
2066
+ "learning_rate": 7.832258291619043e-05,
2067
+ "loss": 0.0204,
2068
+ "step": 3430
2069
+ },
2070
+ {
2071
+ "grad_norm": 0.13357700407505035,
2072
+ "learning_rate": 7.81861666830322e-05,
2073
+ "loss": 0.0292,
2074
+ "step": 3440
2075
+ },
2076
+ {
2077
+ "grad_norm": 0.16105544567108154,
2078
+ "learning_rate": 7.804944221047097e-05,
2079
+ "loss": 0.0233,
2080
+ "step": 3450
2081
+ },
2082
+ {
2083
+ "grad_norm": 0.10065405815839767,
2084
+ "learning_rate": 7.791241099370364e-05,
2085
+ "loss": 0.021,
2086
+ "step": 3460
2087
+ },
2088
+ {
2089
+ "grad_norm": 0.11704110354185104,
2090
+ "learning_rate": 7.777507453128163e-05,
2091
+ "loss": 0.0197,
2092
+ "step": 3470
2093
+ },
2094
+ {
2095
+ "grad_norm": 0.09754088521003723,
2096
+ "learning_rate": 7.763743432509451e-05,
2097
+ "loss": 0.0177,
2098
+ "step": 3480
2099
+ },
2100
+ {
2101
+ "grad_norm": 0.09270471334457397,
2102
+ "learning_rate": 7.749949188035353e-05,
2103
+ "loss": 0.023,
2104
+ "step": 3490
2105
+ },
2106
+ {
2107
+ "grad_norm": 0.12514866888523102,
2108
+ "learning_rate": 7.736124870557516e-05,
2109
+ "loss": 0.0192,
2110
+ "step": 3500
2111
+ },
2112
+ {
2113
+ "grad_norm": 0.17735812067985535,
2114
+ "learning_rate": 7.722270631256459e-05,
2115
+ "loss": 0.0246,
2116
+ "step": 3510
2117
+ },
2118
+ {
2119
+ "grad_norm": 0.13739928603172302,
2120
+ "learning_rate": 7.708386621639925e-05,
2121
+ "loss": 0.0267,
2122
+ "step": 3520
2123
+ },
2124
+ {
2125
+ "grad_norm": 0.09319963306188583,
2126
+ "learning_rate": 7.694472993541219e-05,
2127
+ "loss": 0.0164,
2128
+ "step": 3530
2129
+ },
2130
+ {
2131
+ "grad_norm": 0.15453414618968964,
2132
+ "learning_rate": 7.680529899117547e-05,
2133
+ "loss": 0.0258,
2134
+ "step": 3540
2135
+ },
2136
+ {
2137
+ "grad_norm": 0.09976519644260406,
2138
+ "learning_rate": 7.666557490848358e-05,
2139
+ "loss": 0.0189,
2140
+ "step": 3550
2141
+ },
2142
+ {
2143
+ "grad_norm": 0.1448899358510971,
2144
+ "learning_rate": 7.65255592153367e-05,
2145
+ "loss": 0.0217,
2146
+ "step": 3560
2147
+ },
2148
+ {
2149
+ "grad_norm": 0.29534968733787537,
2150
+ "learning_rate": 7.638525344292402e-05,
2151
+ "loss": 0.0269,
2152
+ "step": 3570
2153
+ },
2154
+ {
2155
+ "grad_norm": 0.14484824240207672,
2156
+ "learning_rate": 7.624465912560697e-05,
2157
+ "loss": 0.0257,
2158
+ "step": 3580
2159
+ },
2160
+ {
2161
+ "grad_norm": 0.1478380411863327,
2162
+ "learning_rate": 7.610377780090249e-05,
2163
+ "loss": 0.0189,
2164
+ "step": 3590
2165
+ },
2166
+ {
2167
+ "grad_norm": 0.13017632067203522,
2168
+ "learning_rate": 7.596261100946618e-05,
2169
+ "loss": 0.0196,
2170
+ "step": 3600
2171
+ },
2172
+ {
2173
+ "grad_norm": 0.14489924907684326,
2174
+ "learning_rate": 7.582116029507542e-05,
2175
+ "loss": 0.0206,
2176
+ "step": 3610
2177
+ },
2178
+ {
2179
+ "grad_norm": 0.1458485722541809,
2180
+ "learning_rate": 7.56794272046126e-05,
2181
+ "loss": 0.0194,
2182
+ "step": 3620
2183
+ },
2184
+ {
2185
+ "grad_norm": 0.14738769829273224,
2186
+ "learning_rate": 7.55374132880481e-05,
2187
+ "loss": 0.0203,
2188
+ "step": 3630
2189
+ },
2190
+ {
2191
+ "grad_norm": 0.13488487899303436,
2192
+ "learning_rate": 7.539512009842333e-05,
2193
+ "loss": 0.0167,
2194
+ "step": 3640
2195
+ },
2196
+ {
2197
+ "grad_norm": 0.19352062046527863,
2198
+ "learning_rate": 7.525254919183382e-05,
2199
+ "loss": 0.0202,
2200
+ "step": 3650
2201
+ },
2202
+ {
2203
+ "grad_norm": 0.10093115270137787,
2204
+ "learning_rate": 7.510970212741215e-05,
2205
+ "loss": 0.0179,
2206
+ "step": 3660
2207
+ },
2208
+ {
2209
+ "grad_norm": 0.11678976565599442,
2210
+ "learning_rate": 7.496658046731096e-05,
2211
+ "loss": 0.02,
2212
+ "step": 3670
2213
+ },
2214
+ {
2215
+ "grad_norm": 0.12365427613258362,
2216
+ "learning_rate": 7.482318577668578e-05,
2217
+ "loss": 0.0169,
2218
+ "step": 3680
2219
+ },
2220
+ {
2221
+ "grad_norm": 0.12157289683818817,
2222
+ "learning_rate": 7.467951962367796e-05,
2223
+ "loss": 0.0184,
2224
+ "step": 3690
2225
+ },
2226
+ {
2227
+ "grad_norm": 0.14409120380878448,
2228
+ "learning_rate": 7.453558357939755e-05,
2229
+ "loss": 0.018,
2230
+ "step": 3700
2231
+ },
2232
+ {
2233
+ "grad_norm": 0.12636560201644897,
2234
+ "learning_rate": 7.439137921790606e-05,
2235
+ "loss": 0.0228,
2236
+ "step": 3710
2237
+ },
2238
+ {
2239
+ "grad_norm": 0.13573269546031952,
2240
+ "learning_rate": 7.42469081161993e-05,
2241
+ "loss": 0.0207,
2242
+ "step": 3720
2243
+ },
2244
+ {
2245
+ "grad_norm": 0.062106985598802567,
2246
+ "learning_rate": 7.410217185419006e-05,
2247
+ "loss": 0.0184,
2248
+ "step": 3730
2249
+ },
2250
+ {
2251
+ "grad_norm": 0.16812735795974731,
2252
+ "learning_rate": 7.395717201469095e-05,
2253
+ "loss": 0.0238,
2254
+ "step": 3740
2255
+ },
2256
+ {
2257
+ "grad_norm": 0.12150177359580994,
2258
+ "learning_rate": 7.381191018339696e-05,
2259
+ "loss": 0.0217,
2260
+ "step": 3750
2261
+ },
2262
+ {
2263
+ "grad_norm": 0.2145930528640747,
2264
+ "learning_rate": 7.36663879488682e-05,
2265
+ "loss": 0.0163,
2266
+ "step": 3760
2267
+ },
2268
+ {
2269
+ "grad_norm": 0.08534784615039825,
2270
+ "learning_rate": 7.352060690251254e-05,
2271
+ "loss": 0.0174,
2272
+ "step": 3770
2273
+ },
2274
+ {
2275
+ "grad_norm": 0.0786997601389885,
2276
+ "learning_rate": 7.337456863856811e-05,
2277
+ "loss": 0.0173,
2278
+ "step": 3780
2279
+ },
2280
+ {
2281
+ "grad_norm": 0.14394225180149078,
2282
+ "learning_rate": 7.3228274754086e-05,
2283
+ "loss": 0.023,
2284
+ "step": 3790
2285
+ },
2286
+ {
2287
+ "grad_norm": 0.1746888905763626,
2288
+ "learning_rate": 7.308172684891267e-05,
2289
+ "loss": 0.0278,
2290
+ "step": 3800
2291
+ },
2292
+ {
2293
+ "grad_norm": 0.11417494714260101,
2294
+ "learning_rate": 7.293492652567255e-05,
2295
+ "loss": 0.024,
2296
+ "step": 3810
2297
+ },
2298
+ {
2299
+ "grad_norm": 0.11693543940782547,
2300
+ "learning_rate": 7.278787538975043e-05,
2301
+ "loss": 0.0223,
2302
+ "step": 3820
2303
+ },
2304
+ {
2305
+ "grad_norm": 0.12056666612625122,
2306
+ "learning_rate": 7.2640575049274e-05,
2307
+ "loss": 0.0163,
2308
+ "step": 3830
2309
+ },
2310
+ {
2311
+ "grad_norm": 0.11734664440155029,
2312
+ "learning_rate": 7.249302711509616e-05,
2313
+ "loss": 0.0215,
2314
+ "step": 3840
2315
+ },
2316
+ {
2317
+ "grad_norm": 0.10295484960079193,
2318
+ "learning_rate": 7.23452332007775e-05,
2319
+ "loss": 0.0171,
2320
+ "step": 3850
2321
+ },
2322
+ {
2323
+ "grad_norm": 0.11296578496694565,
2324
+ "learning_rate": 7.219719492256858e-05,
2325
+ "loss": 0.0244,
2326
+ "step": 3860
2327
+ },
2328
+ {
2329
+ "grad_norm": 0.12529076635837555,
2330
+ "learning_rate": 7.20489138993923e-05,
2331
+ "loss": 0.023,
2332
+ "step": 3870
2333
+ },
2334
+ {
2335
+ "grad_norm": 0.11730336397886276,
2336
+ "learning_rate": 7.190039175282614e-05,
2337
+ "loss": 0.0193,
2338
+ "step": 3880
2339
+ },
2340
+ {
2341
+ "grad_norm": 0.12242171168327332,
2342
+ "learning_rate": 7.175163010708455e-05,
2343
+ "loss": 0.0223,
2344
+ "step": 3890
2345
+ },
2346
+ {
2347
+ "grad_norm": 0.11755640804767609,
2348
+ "learning_rate": 7.1602630589001e-05,
2349
+ "loss": 0.0198,
2350
+ "step": 3900
2351
+ },
2352
+ {
2353
+ "grad_norm": 0.11875393241643906,
2354
+ "learning_rate": 7.14533948280104e-05,
2355
+ "loss": 0.0163,
2356
+ "step": 3910
2357
+ },
2358
+ {
2359
+ "grad_norm": 0.12509284913539886,
2360
+ "learning_rate": 7.130392445613109e-05,
2361
+ "loss": 0.0183,
2362
+ "step": 3920
2363
+ },
2364
+ {
2365
+ "grad_norm": 0.14473293721675873,
2366
+ "learning_rate": 7.115422110794711e-05,
2367
+ "loss": 0.0205,
2368
+ "step": 3930
2369
+ },
2370
+ {
2371
+ "grad_norm": 0.1699618548154831,
2372
+ "learning_rate": 7.100428642059033e-05,
2373
+ "loss": 0.0221,
2374
+ "step": 3940
2375
+ },
2376
+ {
2377
+ "grad_norm": 0.1720869392156601,
2378
+ "learning_rate": 7.08541220337224e-05,
2379
+ "loss": 0.022,
2380
+ "step": 3950
2381
+ },
2382
+ {
2383
+ "grad_norm": 0.11222980916500092,
2384
+ "learning_rate": 7.070372958951706e-05,
2385
+ "loss": 0.0291,
2386
+ "step": 3960
2387
+ },
2388
+ {
2389
+ "grad_norm": 0.13873277604579926,
2390
+ "learning_rate": 7.055311073264194e-05,
2391
+ "loss": 0.0203,
2392
+ "step": 3970
2393
+ },
2394
+ {
2395
+ "grad_norm": 0.11736611276865005,
2396
+ "learning_rate": 7.040226711024077e-05,
2397
+ "loss": 0.0177,
2398
+ "step": 3980
2399
+ },
2400
+ {
2401
+ "grad_norm": 0.09165987372398376,
2402
+ "learning_rate": 7.02512003719152e-05,
2403
+ "loss": 0.0171,
2404
+ "step": 3990
2405
+ },
2406
+ {
2407
+ "grad_norm": 0.1122891902923584,
2408
+ "learning_rate": 7.00999121697069e-05,
2409
+ "loss": 0.0168,
2410
+ "step": 4000
2411
+ },
2412
+ {
2413
+ "grad_norm": 0.14379985630512238,
2414
+ "learning_rate": 6.99484041580794e-05,
2415
+ "loss": 0.0155,
2416
+ "step": 4010
2417
+ },
2418
+ {
2419
+ "grad_norm": 0.1120666041970253,
2420
+ "learning_rate": 6.979667799390004e-05,
2421
+ "loss": 0.015,
2422
+ "step": 4020
2423
+ },
2424
+ {
2425
+ "grad_norm": 0.13411420583724976,
2426
+ "learning_rate": 6.964473533642185e-05,
2427
+ "loss": 0.0176,
2428
+ "step": 4030
2429
+ },
2430
+ {
2431
+ "grad_norm": 0.16511785984039307,
2432
+ "learning_rate": 6.949257784726539e-05,
2433
+ "loss": 0.0225,
2434
+ "step": 4040
2435
+ },
2436
+ {
2437
+ "grad_norm": 0.16178478300571442,
2438
+ "learning_rate": 6.934020719040056e-05,
2439
+ "loss": 0.0211,
2440
+ "step": 4050
2441
+ },
2442
+ {
2443
+ "grad_norm": 0.0941905677318573,
2444
+ "learning_rate": 6.918762503212848e-05,
2445
+ "loss": 0.021,
2446
+ "step": 4060
2447
+ },
2448
+ {
2449
+ "grad_norm": 0.1833467185497284,
2450
+ "learning_rate": 6.903483304106319e-05,
2451
+ "loss": 0.0279,
2452
+ "step": 4070
2453
+ },
2454
+ {
2455
+ "grad_norm": 0.12390021234750748,
2456
+ "learning_rate": 6.888183288811341e-05,
2457
+ "loss": 0.0195,
2458
+ "step": 4080
2459
+ },
2460
+ {
2461
+ "grad_norm": 0.1076701432466507,
2462
+ "learning_rate": 6.87286262464643e-05,
2463
+ "loss": 0.0198,
2464
+ "step": 4090
2465
+ },
2466
+ {
2467
+ "grad_norm": 0.12085499614477158,
2468
+ "learning_rate": 6.857521479155915e-05,
2469
+ "loss": 0.0183,
2470
+ "step": 4100
2471
+ },
2472
+ {
2473
+ "grad_norm": 0.11408424377441406,
2474
+ "learning_rate": 6.842160020108104e-05,
2475
+ "loss": 0.0162,
2476
+ "step": 4110
2477
+ },
2478
+ {
2479
+ "grad_norm": 0.10348336398601532,
2480
+ "learning_rate": 6.826778415493455e-05,
2481
+ "loss": 0.0168,
2482
+ "step": 4120
2483
+ },
2484
+ {
2485
+ "grad_norm": 0.14931976795196533,
2486
+ "learning_rate": 6.811376833522729e-05,
2487
+ "loss": 0.0149,
2488
+ "step": 4130
2489
+ },
2490
+ {
2491
+ "grad_norm": 0.12067539244890213,
2492
+ "learning_rate": 6.795955442625159e-05,
2493
+ "loss": 0.0183,
2494
+ "step": 4140
2495
+ },
2496
+ {
2497
+ "grad_norm": 0.09142495691776276,
2498
+ "learning_rate": 6.780514411446608e-05,
2499
+ "loss": 0.0165,
2500
+ "step": 4150
2501
+ },
2502
+ {
2503
+ "grad_norm": 0.11517301201820374,
2504
+ "learning_rate": 6.765053908847716e-05,
2505
+ "loss": 0.0145,
2506
+ "step": 4160
2507
+ },
2508
+ {
2509
+ "grad_norm": 0.09203200042247772,
2510
+ "learning_rate": 6.749574103902064e-05,
2511
+ "loss": 0.0153,
2512
+ "step": 4170
2513
+ },
2514
+ {
2515
+ "grad_norm": 0.12721046805381775,
2516
+ "learning_rate": 6.734075165894317e-05,
2517
+ "loss": 0.0181,
2518
+ "step": 4180
2519
+ },
2520
+ {
2521
+ "grad_norm": 0.0939367413520813,
2522
+ "learning_rate": 6.71855726431838e-05,
2523
+ "loss": 0.0178,
2524
+ "step": 4190
2525
+ },
2526
+ {
2527
+ "grad_norm": 0.14988242089748383,
2528
+ "learning_rate": 6.703020568875538e-05,
2529
+ "loss": 0.0142,
2530
+ "step": 4200
2531
+ },
2532
+ {
2533
+ "grad_norm": 0.10384884476661682,
2534
+ "learning_rate": 6.687465249472603e-05,
2535
+ "loss": 0.016,
2536
+ "step": 4210
2537
+ },
2538
+ {
2539
+ "grad_norm": 0.07902251929044724,
2540
+ "learning_rate": 6.671891476220055e-05,
2541
+ "loss": 0.0163,
2542
+ "step": 4220
2543
+ },
2544
+ {
2545
+ "grad_norm": 0.11719082295894623,
2546
+ "learning_rate": 6.656299419430183e-05,
2547
+ "loss": 0.0162,
2548
+ "step": 4230
2549
+ },
2550
+ {
2551
+ "grad_norm": 0.11947415769100189,
2552
+ "learning_rate": 6.640689249615223e-05,
2553
+ "loss": 0.0144,
2554
+ "step": 4240
2555
+ },
2556
+ {
2557
+ "grad_norm": 0.12006451189517975,
2558
+ "learning_rate": 6.625061137485491e-05,
2559
+ "loss": 0.0211,
2560
+ "step": 4250
2561
+ },
2562
+ {
2563
+ "grad_norm": 0.10241923481225967,
2564
+ "learning_rate": 6.609415253947517e-05,
2565
+ "loss": 0.0138,
2566
+ "step": 4260
2567
+ },
2568
+ {
2569
+ "grad_norm": 0.1031462550163269,
2570
+ "learning_rate": 6.593751770102178e-05,
2571
+ "loss": 0.0181,
2572
+ "step": 4270
2573
+ },
2574
+ {
2575
+ "grad_norm": 0.13527601957321167,
2576
+ "learning_rate": 6.578070857242823e-05,
2577
+ "loss": 0.0186,
2578
+ "step": 4280
2579
+ },
2580
+ {
2581
+ "grad_norm": 0.19854587316513062,
2582
+ "learning_rate": 6.562372686853402e-05,
2583
+ "loss": 0.0294,
2584
+ "step": 4290
2585
+ },
2586
+ {
2587
+ "grad_norm": 0.10074352473020554,
2588
+ "learning_rate": 6.546657430606593e-05,
2589
+ "loss": 0.0157,
2590
+ "step": 4300
2591
+ },
2592
+ {
2593
+ "grad_norm": 0.13401925563812256,
2594
+ "learning_rate": 6.530925260361918e-05,
2595
+ "loss": 0.0175,
2596
+ "step": 4310
2597
+ },
2598
+ {
2599
+ "grad_norm": 0.1723649501800537,
2600
+ "learning_rate": 6.515176348163871e-05,
2601
+ "loss": 0.0158,
2602
+ "step": 4320
2603
+ },
2604
+ {
2605
+ "grad_norm": 0.16466432809829712,
2606
+ "learning_rate": 6.499410866240032e-05,
2607
+ "loss": 0.0183,
2608
+ "step": 4330
2609
+ },
2610
+ {
2611
+ "grad_norm": 0.09429506212472916,
2612
+ "learning_rate": 6.48362898699919e-05,
2613
+ "loss": 0.0131,
2614
+ "step": 4340
2615
+ },
2616
+ {
2617
+ "grad_norm": 0.11094179004430771,
2618
+ "learning_rate": 6.467830883029443e-05,
2619
+ "loss": 0.0148,
2620
+ "step": 4350
2621
+ },
2622
+ {
2623
+ "grad_norm": 0.1540028154850006,
2624
+ "learning_rate": 6.452016727096326e-05,
2625
+ "loss": 0.0246,
2626
+ "step": 4360
2627
+ },
2628
+ {
2629
+ "grad_norm": 0.13154880702495575,
2630
+ "learning_rate": 6.436186692140916e-05,
2631
+ "loss": 0.0161,
2632
+ "step": 4370
2633
+ },
2634
+ {
2635
+ "grad_norm": 0.09090028703212738,
2636
+ "learning_rate": 6.420340951277938e-05,
2637
+ "loss": 0.018,
2638
+ "step": 4380
2639
+ },
2640
+ {
2641
+ "grad_norm": 0.08051954954862595,
2642
+ "learning_rate": 6.404479677793874e-05,
2643
+ "loss": 0.0214,
2644
+ "step": 4390
2645
+ },
2646
+ {
2647
+ "grad_norm": 0.06494958698749542,
2648
+ "learning_rate": 6.388603045145075e-05,
2649
+ "loss": 0.0128,
2650
+ "step": 4400
2651
+ },
2652
+ {
2653
+ "grad_norm": 0.10332348197698593,
2654
+ "learning_rate": 6.372711226955843e-05,
2655
+ "loss": 0.0155,
2656
+ "step": 4410
2657
+ },
2658
+ {
2659
+ "grad_norm": 0.157622292637825,
2660
+ "learning_rate": 6.356804397016564e-05,
2661
+ "loss": 0.0155,
2662
+ "step": 4420
2663
+ },
2664
+ {
2665
+ "grad_norm": 0.08613033592700958,
2666
+ "learning_rate": 6.340882729281779e-05,
2667
+ "loss": 0.0134,
2668
+ "step": 4430
2669
+ },
2670
+ {
2671
+ "grad_norm": 0.12358319014310837,
2672
+ "learning_rate": 6.324946397868294e-05,
2673
+ "loss": 0.0179,
2674
+ "step": 4440
2675
+ },
2676
+ {
2677
+ "grad_norm": 0.1570926010608673,
2678
+ "learning_rate": 6.308995577053276e-05,
2679
+ "loss": 0.0175,
2680
+ "step": 4450
2681
+ },
2682
+ {
2683
+ "grad_norm": 0.20264583826065063,
2684
+ "learning_rate": 6.293030441272347e-05,
2685
+ "loss": 0.0205,
2686
+ "step": 4460
2687
+ },
2688
+ {
2689
+ "grad_norm": 0.11582266539335251,
2690
+ "learning_rate": 6.277051165117677e-05,
2691
+ "loss": 0.017,
2692
+ "step": 4470
2693
+ },
2694
+ {
2695
+ "grad_norm": 0.1089734211564064,
2696
+ "learning_rate": 6.261057923336064e-05,
2697
+ "loss": 0.0137,
2698
+ "step": 4480
2699
+ },
2700
+ {
2701
+ "grad_norm": 0.10029280930757523,
2702
+ "learning_rate": 6.245050890827042e-05,
2703
+ "loss": 0.0149,
2704
+ "step": 4490
2705
+ },
2706
+ {
2707
+ "grad_norm": 0.10253600031137466,
2708
+ "learning_rate": 6.229030242640952e-05,
2709
+ "loss": 0.0245,
2710
+ "step": 4500
2711
+ },
2712
+ {
2713
+ "grad_norm": 0.14356465637683868,
2714
+ "learning_rate": 6.212996153977037e-05,
2715
+ "loss": 0.0149,
2716
+ "step": 4510
2717
+ },
2718
+ {
2719
+ "grad_norm": 0.12464722245931625,
2720
+ "learning_rate": 6.196948800181523e-05,
2721
+ "loss": 0.022,
2722
+ "step": 4520
2723
+ },
2724
+ {
2725
+ "grad_norm": 0.09775647521018982,
2726
+ "learning_rate": 6.180888356745695e-05,
2727
+ "loss": 0.0163,
2728
+ "step": 4530
2729
+ },
2730
+ {
2731
+ "grad_norm": 0.096392922103405,
2732
+ "learning_rate": 6.164814999303995e-05,
2733
+ "loss": 0.0136,
2734
+ "step": 4540
2735
+ },
2736
+ {
2737
+ "grad_norm": 0.10405683517456055,
2738
+ "learning_rate": 6.148728903632081e-05,
2739
+ "loss": 0.0223,
2740
+ "step": 4550
2741
+ },
2742
+ {
2743
+ "grad_norm": 0.11791567504405975,
2744
+ "learning_rate": 6.132630245644921e-05,
2745
+ "loss": 0.0168,
2746
+ "step": 4560
2747
+ },
2748
+ {
2749
+ "grad_norm": 0.08310893923044205,
2750
+ "learning_rate": 6.116519201394857e-05,
2751
+ "loss": 0.0164,
2752
+ "step": 4570
2753
+ },
2754
+ {
2755
+ "grad_norm": 0.08378935605287552,
2756
+ "learning_rate": 6.10039594706969e-05,
2757
+ "loss": 0.0173,
2758
+ "step": 4580
2759
+ },
2760
+ {
2761
+ "grad_norm": 0.0762941837310791,
2762
+ "learning_rate": 6.084260658990744e-05,
2763
+ "loss": 0.0144,
2764
+ "step": 4590
2765
+ },
2766
+ {
2767
+ "grad_norm": 0.08985130488872528,
2768
+ "learning_rate": 6.068113513610943e-05,
2769
+ "loss": 0.0145,
2770
+ "step": 4600
2771
+ },
2772
+ {
2773
+ "grad_norm": 0.11911336332559586,
2774
+ "learning_rate": 6.0519546875128876e-05,
2775
+ "loss": 0.0215,
2776
+ "step": 4610
2777
+ },
2778
+ {
2779
+ "grad_norm": 0.08101450651884079,
2780
+ "learning_rate": 6.035784357406906e-05,
2781
+ "loss": 0.0191,
2782
+ "step": 4620
2783
+ },
2784
+ {
2785
+ "grad_norm": 0.08633100986480713,
2786
+ "learning_rate": 6.01960270012914e-05,
2787
+ "loss": 0.0129,
2788
+ "step": 4630
2789
+ },
2790
+ {
2791
+ "grad_norm": 0.1489562839269638,
2792
+ "learning_rate": 6.003409892639599e-05,
2793
+ "loss": 0.0166,
2794
+ "step": 4640
2795
+ },
2796
+ {
2797
+ "grad_norm": 0.16843366622924805,
2798
+ "learning_rate": 5.9872061120202336e-05,
2799
+ "loss": 0.0227,
2800
+ "step": 4650
2801
+ },
2802
+ {
2803
+ "grad_norm": 0.1730397492647171,
2804
+ "learning_rate": 5.9709915354729914e-05,
2805
+ "loss": 0.0188,
2806
+ "step": 4660
2807
+ },
2808
+ {
2809
+ "grad_norm": 0.10702052712440491,
2810
+ "learning_rate": 5.9547663403178824e-05,
2811
+ "loss": 0.0175,
2812
+ "step": 4670
2813
+ },
2814
+ {
2815
+ "grad_norm": 0.13497740030288696,
2816
+ "learning_rate": 5.9385307039910445e-05,
2817
+ "loss": 0.0132,
2818
+ "step": 4680
2819
+ },
2820
+ {
2821
+ "grad_norm": 0.08305234462022781,
2822
+ "learning_rate": 5.922284804042792e-05,
2823
+ "loss": 0.02,
2824
+ "step": 4690
2825
+ },
2826
+ {
2827
+ "grad_norm": 0.13063625991344452,
2828
+ "learning_rate": 5.906028818135687e-05,
2829
+ "loss": 0.02,
2830
+ "step": 4700
2831
+ },
2832
+ {
2833
+ "grad_norm": 0.10262671113014221,
2834
+ "learning_rate": 5.889762924042585e-05,
2835
+ "loss": 0.0164,
2836
+ "step": 4710
2837
+ },
2838
+ {
2839
+ "grad_norm": 0.09967511147260666,
2840
+ "learning_rate": 5.873487299644699e-05,
2841
+ "loss": 0.0147,
2842
+ "step": 4720
2843
+ },
2844
+ {
2845
+ "grad_norm": 0.1044909656047821,
2846
+ "learning_rate": 5.857202122929649e-05,
2847
+ "loss": 0.0128,
2848
+ "step": 4730
2849
+ },
2850
+ {
2851
+ "grad_norm": 0.09511242061853409,
2852
+ "learning_rate": 5.840907571989518e-05,
2853
+ "loss": 0.0207,
2854
+ "step": 4740
2855
+ },
2856
+ {
2857
+ "grad_norm": 0.10167711228132248,
2858
+ "learning_rate": 5.824603825018904e-05,
2859
+ "loss": 0.0146,
2860
+ "step": 4750
2861
+ },
2862
+ {
2863
+ "grad_norm": 0.123440220952034,
2864
+ "learning_rate": 5.808291060312975e-05,
2865
+ "loss": 0.0172,
2866
+ "step": 4760
2867
+ },
2868
+ {
2869
+ "grad_norm": 0.1454525589942932,
2870
+ "learning_rate": 5.7919694562655083e-05,
2871
+ "loss": 0.0197,
2872
+ "step": 4770
2873
+ },
2874
+ {
2875
+ "grad_norm": 0.14693108201026917,
2876
+ "learning_rate": 5.775639191366954e-05,
2877
+ "loss": 0.0182,
2878
+ "step": 4780
2879
+ },
2880
+ {
2881
+ "grad_norm": 0.11754677444696426,
2882
+ "learning_rate": 5.75930044420247e-05,
2883
+ "loss": 0.0232,
2884
+ "step": 4790
2885
+ },
2886
+ {
2887
+ "grad_norm": 0.06257650256156921,
2888
+ "learning_rate": 5.74295339344998e-05,
2889
+ "loss": 0.0148,
2890
+ "step": 4800
2891
+ },
2892
+ {
2893
+ "grad_norm": 0.10418052226305008,
2894
+ "learning_rate": 5.726598217878211e-05,
2895
+ "loss": 0.0163,
2896
+ "step": 4810
2897
+ },
2898
+ {
2899
+ "grad_norm": 0.0884476900100708,
2900
+ "learning_rate": 5.71023509634474e-05,
2901
+ "loss": 0.015,
2902
+ "step": 4820
2903
+ },
2904
+ {
2905
+ "grad_norm": 0.1937217265367508,
2906
+ "learning_rate": 5.693864207794049e-05,
2907
+ "loss": 0.017,
2908
+ "step": 4830
2909
+ },
2910
+ {
2911
+ "grad_norm": 0.11750844120979309,
2912
+ "learning_rate": 5.677485731255545e-05,
2913
+ "loss": 0.0154,
2914
+ "step": 4840
2915
+ },
2916
+ {
2917
+ "grad_norm": 0.09092950820922852,
2918
+ "learning_rate": 5.6610998458416296e-05,
2919
+ "loss": 0.0144,
2920
+ "step": 4850
2921
+ },
2922
+ {
2923
+ "grad_norm": 0.10875654965639114,
2924
+ "learning_rate": 5.644706730745716e-05,
2925
+ "loss": 0.016,
2926
+ "step": 4860
2927
+ },
2928
+ {
2929
+ "grad_norm": 0.1630352884531021,
2930
+ "learning_rate": 5.628306565240287e-05,
2931
+ "loss": 0.0201,
2932
+ "step": 4870
2933
+ },
2934
+ {
2935
+ "grad_norm": 0.08496683835983276,
2936
+ "learning_rate": 5.611899528674923e-05,
2937
+ "loss": 0.0173,
2938
+ "step": 4880
2939
+ },
2940
+ {
2941
+ "grad_norm": 0.11503885686397552,
2942
+ "learning_rate": 5.595485800474349e-05,
2943
+ "loss": 0.0152,
2944
+ "step": 4890
2945
+ },
2946
+ {
2947
+ "grad_norm": 0.1367446631193161,
2948
+ "learning_rate": 5.579065560136467e-05,
2949
+ "loss": 0.0149,
2950
+ "step": 4900
2951
+ },
2952
+ {
2953
+ "grad_norm": 0.12649130821228027,
2954
+ "learning_rate": 5.562638987230392e-05,
2955
+ "loss": 0.0183,
2956
+ "step": 4910
2957
+ },
2958
+ {
2959
+ "grad_norm": 0.09708788245916367,
2960
+ "learning_rate": 5.546206261394498e-05,
2961
+ "loss": 0.0191,
2962
+ "step": 4920
2963
+ },
2964
+ {
2965
+ "grad_norm": 0.0999777540564537,
2966
+ "learning_rate": 5.529767562334437e-05,
2967
+ "loss": 0.0168,
2968
+ "step": 4930
2969
+ },
2970
+ {
2971
+ "grad_norm": 0.12223142385482788,
2972
+ "learning_rate": 5.5133230698211926e-05,
2973
+ "loss": 0.0155,
2974
+ "step": 4940
2975
+ },
2976
+ {
2977
+ "grad_norm": 0.09065050631761551,
2978
+ "learning_rate": 5.496872963689096e-05,
2979
+ "loss": 0.0132,
2980
+ "step": 4950
2981
+ },
2982
+ {
2983
+ "grad_norm": 0.146714985370636,
2984
+ "learning_rate": 5.4804174238338756e-05,
2985
+ "loss": 0.0209,
2986
+ "step": 4960
2987
+ },
2988
+ {
2989
+ "grad_norm": 0.11266348510980606,
2990
+ "learning_rate": 5.463956630210678e-05,
2991
+ "loss": 0.0139,
2992
+ "step": 4970
2993
+ },
2994
+ {
2995
+ "grad_norm": 0.07989142090082169,
2996
+ "learning_rate": 5.4474907628321046e-05,
2997
+ "loss": 0.011,
2998
+ "step": 4980
2999
+ },
3000
+ {
3001
+ "grad_norm": 0.14550931751728058,
3002
+ "learning_rate": 5.431020001766244e-05,
3003
+ "loss": 0.0177,
3004
+ "step": 4990
3005
+ },
3006
+ {
3007
+ "grad_norm": 0.1117406040430069,
3008
+ "learning_rate": 5.4145445271346986e-05,
3009
+ "loss": 0.0178,
3010
+ "step": 5000
3011
+ },
3012
+ {
3013
+ "grad_norm": 0.12149982154369354,
3014
+ "learning_rate": 5.398064519110622e-05,
3015
+ "loss": 0.0175,
3016
+ "step": 5010
3017
+ },
3018
+ {
3019
+ "grad_norm": 0.21780666708946228,
3020
+ "learning_rate": 5.3815801579167394e-05,
3021
+ "loss": 0.0167,
3022
+ "step": 5020
3023
+ },
3024
+ {
3025
+ "grad_norm": 0.17102351784706116,
3026
+ "learning_rate": 5.365091623823382e-05,
3027
+ "loss": 0.0175,
3028
+ "step": 5030
3029
+ },
3030
+ {
3031
+ "grad_norm": 0.11842578649520874,
3032
+ "learning_rate": 5.348599097146521e-05,
3033
+ "loss": 0.0128,
3034
+ "step": 5040
3035
+ },
3036
+ {
3037
+ "grad_norm": 0.1212945505976677,
3038
+ "learning_rate": 5.3321027582457836e-05,
3039
+ "loss": 0.0156,
3040
+ "step": 5050
3041
+ },
3042
+ {
3043
+ "grad_norm": 0.09221485257148743,
3044
+ "learning_rate": 5.315602787522491e-05,
3045
+ "loss": 0.012,
3046
+ "step": 5060
3047
+ },
3048
+ {
3049
+ "grad_norm": 0.094135582447052,
3050
+ "learning_rate": 5.299099365417678e-05,
3051
+ "loss": 0.0155,
3052
+ "step": 5070
3053
+ },
3054
+ {
3055
+ "grad_norm": 0.11543729156255722,
3056
+ "learning_rate": 5.2825926724101236e-05,
3057
+ "loss": 0.0182,
3058
+ "step": 5080
3059
+ },
3060
+ {
3061
+ "grad_norm": 0.17128892242908478,
3062
+ "learning_rate": 5.26608288901438e-05,
3063
+ "loss": 0.018,
3064
+ "step": 5090
3065
+ },
3066
+ {
3067
+ "grad_norm": 0.12547394633293152,
3068
+ "learning_rate": 5.24957019577879e-05,
3069
+ "loss": 0.0215,
3070
+ "step": 5100
3071
+ },
3072
+ {
3073
+ "grad_norm": 0.21015821397304535,
3074
+ "learning_rate": 5.2330547732835266e-05,
3075
+ "loss": 0.0166,
3076
+ "step": 5110
3077
+ },
3078
+ {
3079
+ "grad_norm": 0.34002071619033813,
3080
+ "learning_rate": 5.2165368021385996e-05,
3081
+ "loss": 0.0179,
3082
+ "step": 5120
3083
+ },
3084
+ {
3085
+ "grad_norm": 0.14867143332958221,
3086
+ "learning_rate": 5.200016462981897e-05,
3087
+ "loss": 0.0145,
3088
+ "step": 5130
3089
+ },
3090
+ {
3091
+ "grad_norm": 0.09806505590677261,
3092
+ "learning_rate": 5.1834939364772015e-05,
3093
+ "loss": 0.0155,
3094
+ "step": 5140
3095
+ },
3096
+ {
3097
+ "grad_norm": 0.14923235774040222,
3098
+ "learning_rate": 5.166969403312214e-05,
3099
+ "loss": 0.018,
3100
+ "step": 5150
3101
+ },
3102
+ {
3103
+ "grad_norm": 0.1007002517580986,
3104
+ "learning_rate": 5.1504430441965844e-05,
3105
+ "loss": 0.015,
3106
+ "step": 5160
3107
+ },
3108
+ {
3109
+ "grad_norm": 0.19337940216064453,
3110
+ "learning_rate": 5.133915039859923e-05,
3111
+ "loss": 0.0164,
3112
+ "step": 5170
3113
+ },
3114
+ {
3115
+ "grad_norm": 0.0948144942522049,
3116
+ "learning_rate": 5.1173855710498444e-05,
3117
+ "loss": 0.0177,
3118
+ "step": 5180
3119
+ },
3120
+ {
3121
+ "grad_norm": 0.09900026768445969,
3122
+ "learning_rate": 5.100854818529967e-05,
3123
+ "loss": 0.0146,
3124
+ "step": 5190
3125
+ },
3126
+ {
3127
+ "grad_norm": 0.10444221645593643,
3128
+ "learning_rate": 5.084322963077951e-05,
3129
+ "loss": 0.0155,
3130
+ "step": 5200
3131
+ },
3132
+ {
3133
+ "grad_norm": 0.0999322310090065,
3134
+ "learning_rate": 5.067790185483522e-05,
3135
+ "loss": 0.0172,
3136
+ "step": 5210
3137
+ },
3138
+ {
3139
+ "grad_norm": 0.13305051624774933,
3140
+ "learning_rate": 5.0512566665464844e-05,
3141
+ "loss": 0.0128,
3142
+ "step": 5220
3143
+ },
3144
+ {
3145
+ "grad_norm": 0.21085679531097412,
3146
+ "learning_rate": 5.034722587074755e-05,
3147
+ "loss": 0.0192,
3148
+ "step": 5230
3149
+ },
3150
+ {
3151
+ "grad_norm": 0.10392367094755173,
3152
+ "learning_rate": 5.018188127882375e-05,
3153
+ "loss": 0.0139,
3154
+ "step": 5240
3155
+ },
3156
+ {
3157
+ "grad_norm": 0.119481660425663,
3158
+ "learning_rate": 5.0016534697875417e-05,
3159
+ "loss": 0.0125,
3160
+ "step": 5250
3161
+ },
3162
+ {
3163
+ "grad_norm": 0.11169359087944031,
3164
+ "learning_rate": 4.9851187936106294e-05,
3165
+ "loss": 0.0159,
3166
+ "step": 5260
3167
+ },
3168
+ {
3169
+ "grad_norm": 0.0979238748550415,
3170
+ "learning_rate": 4.968584280172206e-05,
3171
+ "loss": 0.0155,
3172
+ "step": 5270
3173
+ },
3174
+ {
3175
+ "grad_norm": 0.1328539103269577,
3176
+ "learning_rate": 4.95205011029106e-05,
3177
+ "loss": 0.0108,
3178
+ "step": 5280
3179
+ },
3180
+ {
3181
+ "grad_norm": 0.11174710094928741,
3182
+ "learning_rate": 4.935516464782227e-05,
3183
+ "loss": 0.0142,
3184
+ "step": 5290
3185
+ },
3186
+ {
3187
+ "grad_norm": 0.1004805713891983,
3188
+ "learning_rate": 4.918983524455003e-05,
3189
+ "loss": 0.0194,
3190
+ "step": 5300
3191
+ },
3192
+ {
3193
+ "grad_norm": 0.13593004643917084,
3194
+ "learning_rate": 4.9024514701109766e-05,
3195
+ "loss": 0.028,
3196
+ "step": 5310
3197
+ },
3198
+ {
3199
+ "grad_norm": 0.12582026422023773,
3200
+ "learning_rate": 4.885920482542043e-05,
3201
+ "loss": 0.0166,
3202
+ "step": 5320
3203
+ },
3204
+ {
3205
+ "grad_norm": 0.11316690593957901,
3206
+ "learning_rate": 4.869390742528438e-05,
3207
+ "loss": 0.0195,
3208
+ "step": 5330
3209
+ },
3210
+ {
3211
+ "grad_norm": 0.07854901999235153,
3212
+ "learning_rate": 4.852862430836744e-05,
3213
+ "loss": 0.0137,
3214
+ "step": 5340
3215
+ },
3216
+ {
3217
+ "grad_norm": 0.1230713501572609,
3218
+ "learning_rate": 4.836335728217933e-05,
3219
+ "loss": 0.0207,
3220
+ "step": 5350
3221
+ },
3222
+ {
3223
+ "grad_norm": 0.11357402056455612,
3224
+ "learning_rate": 4.819810815405379e-05,
3225
+ "loss": 0.0168,
3226
+ "step": 5360
3227
+ },
3228
+ {
3229
+ "grad_norm": 0.1529230773448944,
3230
+ "learning_rate": 4.803287873112877e-05,
3231
+ "loss": 0.0116,
3232
+ "step": 5370
3233
+ },
3234
+ {
3235
+ "grad_norm": 0.08224724233150482,
3236
+ "learning_rate": 4.786767082032681e-05,
3237
+ "loss": 0.0189,
3238
+ "step": 5380
3239
+ },
3240
+ {
3241
+ "grad_norm": 0.17682884633541107,
3242
+ "learning_rate": 4.77024862283351e-05,
3243
+ "loss": 0.0183,
3244
+ "step": 5390
3245
+ },
3246
+ {
3247
+ "grad_norm": 0.09716563671827316,
3248
+ "learning_rate": 4.753732676158593e-05,
3249
+ "loss": 0.0174,
3250
+ "step": 5400
3251
+ },
3252
+ {
3253
+ "grad_norm": 0.08129530400037766,
3254
+ "learning_rate": 4.737219422623672e-05,
3255
+ "loss": 0.0139,
3256
+ "step": 5410
3257
+ },
3258
+ {
3259
+ "grad_norm": 0.12539704144001007,
3260
+ "learning_rate": 4.720709042815044e-05,
3261
+ "loss": 0.0182,
3262
+ "step": 5420
3263
+ },
3264
+ {
3265
+ "grad_norm": 0.09867323935031891,
3266
+ "learning_rate": 4.704201717287578e-05,
3267
+ "loss": 0.0122,
3268
+ "step": 5430
3269
+ },
3270
+ {
3271
+ "grad_norm": 0.10833761841058731,
3272
+ "learning_rate": 4.6876976265627404e-05,
3273
+ "loss": 0.0164,
3274
+ "step": 5440
3275
+ },
3276
+ {
3277
+ "grad_norm": 0.11536791175603867,
3278
+ "learning_rate": 4.671196951126626e-05,
3279
+ "loss": 0.0153,
3280
+ "step": 5450
3281
+ },
3282
+ {
3283
+ "grad_norm": 0.09528806805610657,
3284
+ "learning_rate": 4.654699871427971e-05,
3285
+ "loss": 0.0132,
3286
+ "step": 5460
3287
+ },
3288
+ {
3289
+ "grad_norm": 0.15388181805610657,
3290
+ "learning_rate": 4.6382065678762034e-05,
3291
+ "loss": 0.0198,
3292
+ "step": 5470
3293
+ },
3294
+ {
3295
+ "grad_norm": 0.08998192846775055,
3296
+ "learning_rate": 4.6217172208394424e-05,
3297
+ "loss": 0.0109,
3298
+ "step": 5480
3299
+ },
3300
+ {
3301
+ "grad_norm": 0.07558619976043701,
3302
+ "learning_rate": 4.605232010642549e-05,
3303
+ "loss": 0.0119,
3304
+ "step": 5490
3305
+ },
3306
+ {
3307
+ "grad_norm": 0.10081010311841965,
3308
+ "learning_rate": 4.588751117565142e-05,
3309
+ "loss": 0.0131,
3310
+ "step": 5500
3311
+ },
3312
+ {
3313
+ "grad_norm": 0.12107884138822556,
3314
+ "learning_rate": 4.5722747218396214e-05,
3315
+ "loss": 0.0146,
3316
+ "step": 5510
3317
+ },
3318
+ {
3319
+ "grad_norm": 0.11925487220287323,
3320
+ "learning_rate": 4.5558030036492194e-05,
3321
+ "loss": 0.0135,
3322
+ "step": 5520
3323
+ },
3324
+ {
3325
+ "grad_norm": 0.12009993940591812,
3326
+ "learning_rate": 4.539336143125999e-05,
3327
+ "loss": 0.0142,
3328
+ "step": 5530
3329
+ },
3330
+ {
3331
+ "grad_norm": 0.06643348187208176,
3332
+ "learning_rate": 4.522874320348916e-05,
3333
+ "loss": 0.0134,
3334
+ "step": 5540
3335
+ },
3336
+ {
3337
+ "grad_norm": 0.1483471393585205,
3338
+ "learning_rate": 4.506417715341821e-05,
3339
+ "loss": 0.0134,
3340
+ "step": 5550
3341
+ },
3342
+ {
3343
+ "grad_norm": 0.14901196956634521,
3344
+ "learning_rate": 4.489966508071511e-05,
3345
+ "loss": 0.0149,
3346
+ "step": 5560
3347
+ },
3348
+ {
3349
+ "grad_norm": 0.1339421421289444,
3350
+ "learning_rate": 4.4735208784457575e-05,
3351
+ "loss": 0.0154,
3352
+ "step": 5570
3353
+ },
3354
+ {
3355
+ "grad_norm": 0.09905123710632324,
3356
+ "learning_rate": 4.457081006311325e-05,
3357
+ "loss": 0.0111,
3358
+ "step": 5580
3359
+ },
3360
+ {
3361
+ "grad_norm": 0.08451845496892929,
3362
+ "learning_rate": 4.440647071452027e-05,
3363
+ "loss": 0.0125,
3364
+ "step": 5590
3365
+ },
3366
+ {
3367
+ "grad_norm": 0.10544228553771973,
3368
+ "learning_rate": 4.424219253586737e-05,
3369
+ "loss": 0.0168,
3370
+ "step": 5600
3371
+ },
3372
+ {
3373
+ "grad_norm": 0.10120836645364761,
3374
+ "learning_rate": 4.407797732367443e-05,
3375
+ "loss": 0.0145,
3376
+ "step": 5610
3377
+ },
3378
+ {
3379
+ "grad_norm": 0.10984590649604797,
3380
+ "learning_rate": 4.391382687377268e-05,
3381
+ "loss": 0.0164,
3382
+ "step": 5620
3383
+ },
3384
+ {
3385
+ "grad_norm": 0.06651951372623444,
3386
+ "learning_rate": 4.374974298128512e-05,
3387
+ "loss": 0.0107,
3388
+ "step": 5630
3389
+ },
3390
+ {
3391
+ "grad_norm": 0.14574792981147766,
3392
+ "learning_rate": 4.358572744060699e-05,
3393
+ "loss": 0.0166,
3394
+ "step": 5640
3395
+ },
3396
+ {
3397
+ "grad_norm": 0.09139572083950043,
3398
+ "learning_rate": 4.342178204538588e-05,
3399
+ "loss": 0.014,
3400
+ "step": 5650
3401
+ },
3402
+ {
3403
+ "grad_norm": 0.17951051890850067,
3404
+ "learning_rate": 4.325790858850241e-05,
3405
+ "loss": 0.0136,
3406
+ "step": 5660
3407
+ },
3408
+ {
3409
+ "grad_norm": 0.10618460178375244,
3410
+ "learning_rate": 4.309410886205043e-05,
3411
+ "loss": 0.0109,
3412
+ "step": 5670
3413
+ },
3414
+ {
3415
+ "grad_norm": 0.10198267549276352,
3416
+ "learning_rate": 4.293038465731752e-05,
3417
+ "loss": 0.0187,
3418
+ "step": 5680
3419
+ },
3420
+ {
3421
+ "grad_norm": 0.12924885749816895,
3422
+ "learning_rate": 4.276673776476533e-05,
3423
+ "loss": 0.0107,
3424
+ "step": 5690
3425
+ },
3426
+ {
3427
+ "grad_norm": 0.13997332751750946,
3428
+ "learning_rate": 4.260316997401007e-05,
3429
+ "loss": 0.0169,
3430
+ "step": 5700
3431
+ },
3432
+ {
3433
+ "grad_norm": 0.07969796657562256,
3434
+ "learning_rate": 4.243968307380293e-05,
3435
+ "loss": 0.0174,
3436
+ "step": 5710
3437
+ },
3438
+ {
3439
+ "grad_norm": 0.10636241734027863,
3440
+ "learning_rate": 4.22762788520104e-05,
3441
+ "loss": 0.011,
3442
+ "step": 5720
3443
+ },
3444
+ {
3445
+ "grad_norm": 0.08587957918643951,
3446
+ "learning_rate": 4.211295909559491e-05,
3447
+ "loss": 0.0146,
3448
+ "step": 5730
3449
+ },
3450
+ {
3451
+ "grad_norm": 0.12792466580867767,
3452
+ "learning_rate": 4.194972559059511e-05,
3453
+ "loss": 0.0104,
3454
+ "step": 5740
3455
+ },
3456
+ {
3457
+ "grad_norm": 0.07166793942451477,
3458
+ "learning_rate": 4.178658012210651e-05,
3459
+ "loss": 0.0139,
3460
+ "step": 5750
3461
+ },
3462
+ {
3463
+ "grad_norm": 0.06277091056108475,
3464
+ "learning_rate": 4.162352447426177e-05,
3465
+ "loss": 0.0108,
3466
+ "step": 5760
3467
+ },
3468
+ {
3469
+ "grad_norm": 0.10091009736061096,
3470
+ "learning_rate": 4.146056043021135e-05,
3471
+ "loss": 0.0106,
3472
+ "step": 5770
3473
+ },
3474
+ {
3475
+ "grad_norm": 0.0645943358540535,
3476
+ "learning_rate": 4.1297689772103944e-05,
3477
+ "loss": 0.0134,
3478
+ "step": 5780
3479
+ },
3480
+ {
3481
+ "grad_norm": 0.10704907774925232,
3482
+ "learning_rate": 4.113491428106694e-05,
3483
+ "loss": 0.0128,
3484
+ "step": 5790
3485
+ },
3486
+ {
3487
+ "grad_norm": 0.10993106663227081,
3488
+ "learning_rate": 4.0972235737187055e-05,
3489
+ "loss": 0.013,
3490
+ "step": 5800
3491
+ },
3492
+ {
3493
+ "grad_norm": 0.1654406189918518,
3494
+ "learning_rate": 4.080965591949076e-05,
3495
+ "loss": 0.0236,
3496
+ "step": 5810
3497
+ },
3498
+ {
3499
+ "grad_norm": 0.12182041257619858,
3500
+ "learning_rate": 4.0647176605924924e-05,
3501
+ "loss": 0.0211,
3502
+ "step": 5820
3503
+ },
3504
+ {
3505
+ "grad_norm": 0.07427161931991577,
3506
+ "learning_rate": 4.0484799573337255e-05,
3507
+ "loss": 0.0154,
3508
+ "step": 5830
3509
+ },
3510
+ {
3511
+ "grad_norm": 0.0983806699514389,
3512
+ "learning_rate": 4.032252659745699e-05,
3513
+ "loss": 0.0128,
3514
+ "step": 5840
3515
+ },
3516
+ {
3517
+ "grad_norm": 0.10781994462013245,
3518
+ "learning_rate": 4.016035945287539e-05,
3519
+ "loss": 0.0129,
3520
+ "step": 5850
3521
+ },
3522
+ {
3523
+ "grad_norm": 0.10114669799804688,
3524
+ "learning_rate": 3.999829991302635e-05,
3525
+ "loss": 0.0145,
3526
+ "step": 5860
3527
+ },
3528
+ {
3529
+ "grad_norm": 0.1134527251124382,
3530
+ "learning_rate": 3.983634975016707e-05,
3531
+ "loss": 0.0125,
3532
+ "step": 5870
3533
+ },
3534
+ {
3535
+ "grad_norm": 0.08050957322120667,
3536
+ "learning_rate": 3.967451073535854e-05,
3537
+ "loss": 0.0166,
3538
+ "step": 5880
3539
+ },
3540
+ {
3541
+ "grad_norm": 0.11550536006689072,
3542
+ "learning_rate": 3.951278463844633e-05,
3543
+ "loss": 0.0126,
3544
+ "step": 5890
3545
+ },
3546
+ {
3547
+ "grad_norm": 0.14811114966869354,
3548
+ "learning_rate": 3.935117322804111e-05,
3549
+ "loss": 0.0147,
3550
+ "step": 5900
3551
+ },
3552
+ {
3553
+ "grad_norm": 0.09880905598402023,
3554
+ "learning_rate": 3.918967827149938e-05,
3555
+ "loss": 0.0161,
3556
+ "step": 5910
3557
+ },
3558
+ {
3559
+ "grad_norm": 0.15333198010921478,
3560
+ "learning_rate": 3.9028301534904094e-05,
3561
+ "loss": 0.0173,
3562
+ "step": 5920
3563
+ },
3564
+ {
3565
+ "grad_norm": 0.12640492618083954,
3566
+ "learning_rate": 3.88670447830454e-05,
3567
+ "loss": 0.0147,
3568
+ "step": 5930
3569
+ },
3570
+ {
3571
+ "grad_norm": 0.163604736328125,
3572
+ "learning_rate": 3.870590977940132e-05,
3573
+ "loss": 0.0171,
3574
+ "step": 5940
3575
+ },
3576
+ {
3577
+ "grad_norm": 0.11071155965328217,
3578
+ "learning_rate": 3.8544898286118404e-05,
3579
+ "loss": 0.0121,
3580
+ "step": 5950
3581
+ },
3582
+ {
3583
+ "grad_norm": 0.07924096286296844,
3584
+ "learning_rate": 3.838401206399257e-05,
3585
+ "loss": 0.0127,
3586
+ "step": 5960
3587
+ },
3588
+ {
3589
+ "grad_norm": 0.07424289733171463,
3590
+ "learning_rate": 3.822325287244975e-05,
3591
+ "loss": 0.0125,
3592
+ "step": 5970
3593
+ },
3594
+ {
3595
+ "grad_norm": 0.08219657093286514,
3596
+ "learning_rate": 3.8062622469526725e-05,
3597
+ "loss": 0.0094,
3598
+ "step": 5980
3599
+ },
3600
+ {
3601
+ "grad_norm": 0.1099315956234932,
3602
+ "learning_rate": 3.790212261185183e-05,
3603
+ "loss": 0.0144,
3604
+ "step": 5990
3605
+ },
3606
+ {
3607
+ "grad_norm": 0.09500293433666229,
3608
+ "learning_rate": 3.7741755054625794e-05,
3609
+ "loss": 0.0167,
3610
+ "step": 6000
3611
+ }
3612
+ ],
3613
+ "logging_steps": 10,
3614
+ "max_steps": 10000,
3615
+ "num_input_tokens_seen": 0,
3616
+ "num_train_epochs": 9223372036854775807,
3617
+ "save_steps": 2000,
3618
+ "stateful_callbacks": {
3619
+ "TrainerControl": {
3620
+ "args": {
3621
+ "should_epoch_stop": false,
3622
+ "should_evaluate": false,
3623
+ "should_log": false,
3624
+ "should_save": true,
3625
+ "should_training_stop": false
3626
+ },
3627
+ "attributes": {}
3628
+ }
3629
+ },
3630
+ "total_flos": 0.0,
3631
+ "train_batch_size": 16,
3632
+ "trial_name": null,
3633
+ "trial_params": null
3634
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b4c4afc66cc9f6d8e2674ddac88de11f488edba49c9c0894007f547e8bd35d98
3
+ size 7633
wandb_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"project": "atomicvla", "entity": "happyhappy", "run_id": "abcd2a_control_ft"}
zero_to_fp32.py ADDED
@@ -0,0 +1,760 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example:
14
+ # python zero_to_fp32.py . output_dir/
15
+ # or
16
+ # python zero_to_fp32.py . output_dir/ --safe_serialization
17
+
18
+ import argparse
19
+ import torch
20
+ import glob
21
+ import math
22
+ import os
23
+ import re
24
+ import gc
25
+ import json
26
+ import numpy as np
27
+ from tqdm import tqdm
28
+ from collections import OrderedDict
29
+ from dataclasses import dataclass
30
+
31
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
+ # DeepSpeed data structures it has to be available in the current python environment.
33
+ from deepspeed.utils import logger
34
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
+
38
+
39
+ @dataclass
40
+ class zero_model_state:
41
+ buffers: dict()
42
+ param_shapes: dict()
43
+ shared_params: list
44
+ ds_version: int
45
+ frozen_param_shapes: dict()
46
+ frozen_param_fragments: dict()
47
+
48
+
49
+ debug = 0
50
+
51
+ # load to cpu
52
+ device = torch.device('cpu')
53
+
54
+
55
+ def atoi(text):
56
+ return int(text) if text.isdigit() else text
57
+
58
+
59
+ def natural_keys(text):
60
+ '''
61
+ alist.sort(key=natural_keys) sorts in human order
62
+ http://nedbatchelder.com/blog/200712/human_sorting.html
63
+ (See Toothy's implementation in the comments)
64
+ '''
65
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
66
+
67
+
68
+ def get_model_state_file(checkpoint_dir, zero_stage):
69
+ if not os.path.isdir(checkpoint_dir):
70
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
+
72
+ # there should be only one file
73
+ if zero_stage <= 2:
74
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
+ elif zero_stage == 3:
76
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
+
78
+ if not os.path.exists(file):
79
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
80
+
81
+ return file
82
+
83
+
84
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
+ # XXX: need to test that this simple glob rule works for multi-node setup too
86
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
+
88
+ if len(ckpt_files) == 0:
89
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
+
91
+ return ckpt_files
92
+
93
+
94
+ def get_optim_files(checkpoint_dir):
95
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
+
97
+
98
+ def get_model_state_files(checkpoint_dir):
99
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
+
101
+
102
+ def parse_model_states(files):
103
+ zero_model_states = []
104
+ for file in files:
105
+ state_dict = torch.load(file, map_location=device, weights_only=False)
106
+
107
+ if BUFFER_NAMES not in state_dict:
108
+ raise ValueError(f"{file} is not a model state checkpoint")
109
+ buffer_names = state_dict[BUFFER_NAMES]
110
+ if debug:
111
+ print("Found buffers:", buffer_names)
112
+
113
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
+ param_shapes = state_dict[PARAM_SHAPES]
116
+
117
+ # collect parameters that are included in param_shapes
118
+ param_names = []
119
+ for s in param_shapes:
120
+ for name in s.keys():
121
+ param_names.append(name)
122
+
123
+ # update with frozen parameters
124
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
+ if frozen_param_shapes is not None:
126
+ if debug:
127
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
+ param_names += list(frozen_param_shapes.keys())
129
+
130
+ # handle shared params
131
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
+
133
+ ds_version = state_dict.get(DS_VERSION, None)
134
+
135
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
+
137
+ z_model_state = zero_model_state(buffers=buffers,
138
+ param_shapes=param_shapes,
139
+ shared_params=shared_params,
140
+ ds_version=ds_version,
141
+ frozen_param_shapes=frozen_param_shapes,
142
+ frozen_param_fragments=frozen_param_fragments)
143
+ zero_model_states.append(z_model_state)
144
+
145
+ return zero_model_states
146
+
147
+
148
+ def parse_optim_states(files, ds_checkpoint_dir):
149
+ total_files = len(files)
150
+ state_dicts = []
151
+ for f in tqdm(files, desc='Loading checkpoint shards'):
152
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
+ # and also handle the case where it was already removed by another helper script
155
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
+ state_dicts.append(state_dict)
157
+
158
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
160
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
+
163
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
165
+ # use the max of the partition_count to get the dp world_size.
166
+
167
+ if type(world_size) is list:
168
+ world_size = max(world_size)
169
+
170
+ if world_size != total_files:
171
+ raise ValueError(
172
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
+ )
175
+
176
+ # the groups are named differently in each stage
177
+ if zero_stage <= 2:
178
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
+ elif zero_stage == 3:
180
+ fp32_groups_key = FP32_FLAT_GROUPS
181
+ else:
182
+ raise ValueError(f"unknown zero stage {zero_stage}")
183
+
184
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
+ return zero_stage, world_size, fp32_flat_groups
186
+
187
+
188
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
+ """
190
+ Returns fp32 state_dict reconstructed from ds checkpoint
191
+
192
+ Args:
193
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
+
195
+ """
196
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
+
198
+ optim_files = get_optim_files(ds_checkpoint_dir)
199
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
+
202
+ model_files = get_model_state_files(ds_checkpoint_dir)
203
+
204
+ zero_model_states = parse_model_states(model_files)
205
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
+
207
+ if zero_stage <= 2:
208
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
+ exclude_frozen_parameters)
210
+ elif zero_stage == 3:
211
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
+ exclude_frozen_parameters)
213
+
214
+
215
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
+ return
218
+
219
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
+
222
+ if debug:
223
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
+
226
+ wanted_params = len(frozen_param_shapes)
227
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
+ print(f'Frozen params: Have {avail_numel} numels to process.')
230
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
+
232
+ total_params = 0
233
+ total_numel = 0
234
+ for name, shape in frozen_param_shapes.items():
235
+ total_params += 1
236
+ unpartitioned_numel = shape.numel()
237
+ total_numel += unpartitioned_numel
238
+
239
+ state_dict[name] = frozen_param_fragments[name]
240
+
241
+ if debug:
242
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
+
244
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
+
246
+
247
+ def _has_callable(obj, fn):
248
+ attr = getattr(obj, fn, None)
249
+ return callable(attr)
250
+
251
+
252
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
+ param_shapes = zero_model_states[0].param_shapes
254
+
255
+ # Reconstruction protocol:
256
+ #
257
+ # XXX: document this
258
+
259
+ if debug:
260
+ for i in range(world_size):
261
+ for j in range(len(fp32_flat_groups[0])):
262
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
+
264
+ # XXX: memory usage doubles here (zero2)
265
+ num_param_groups = len(fp32_flat_groups[0])
266
+ merged_single_partition_of_fp32_groups = []
267
+ for i in range(num_param_groups):
268
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
+ avail_numel = sum(
272
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
+
274
+ if debug:
275
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
276
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
+ # not asserting if there is a mismatch due to possible padding
278
+ print(f"Have {avail_numel} numels to process.")
279
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
+
281
+ # params
282
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
+ # out-of-core computing solution
284
+ total_numel = 0
285
+ total_params = 0
286
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
+ offset = 0
288
+ avail_numel = full_single_fp32_vector.numel()
289
+ for name, shape in shapes.items():
290
+
291
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
+ total_numel += unpartitioned_numel
293
+ total_params += 1
294
+
295
+ if debug:
296
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
+ offset += unpartitioned_numel
299
+
300
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
+ # live optimizer object, so we are checking that the numbers are within the right range
304
+ align_to = 2 * world_size
305
+
306
+ def zero2_align(x):
307
+ return align_to * math.ceil(x / align_to)
308
+
309
+ if debug:
310
+ print(f"original offset={offset}, avail_numel={avail_numel}")
311
+
312
+ offset = zero2_align(offset)
313
+ avail_numel = zero2_align(avail_numel)
314
+
315
+ if debug:
316
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
+
318
+ # Sanity check
319
+ if offset != avail_numel:
320
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
+
322
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
+
324
+
325
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
+ exclude_frozen_parameters):
327
+ state_dict = OrderedDict()
328
+
329
+ # buffers
330
+ buffers = zero_model_states[0].buffers
331
+ state_dict.update(buffers)
332
+ if debug:
333
+ print(f"added {len(buffers)} buffers")
334
+
335
+ if not exclude_frozen_parameters:
336
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
337
+
338
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
+
340
+ # recover shared parameters
341
+ for pair in zero_model_states[0].shared_params:
342
+ if pair[1] in state_dict:
343
+ state_dict[pair[0]] = state_dict[pair[1]]
344
+
345
+ return state_dict
346
+
347
+
348
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
+ remainder = unpartitioned_numel % world_size
350
+ padding_numel = (world_size - remainder) if remainder else 0
351
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
+ return partitioned_numel, padding_numel
353
+
354
+
355
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
+ return
358
+
359
+ if debug:
360
+ for i in range(world_size):
361
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
+
364
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
+ wanted_params = len(frozen_param_shapes)
366
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
+ print(f'Frozen params: Have {avail_numel} numels to process.')
369
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
+
371
+ total_params = 0
372
+ total_numel = 0
373
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
+ total_params += 1
375
+ unpartitioned_numel = shape.numel()
376
+ total_numel += unpartitioned_numel
377
+
378
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
+
381
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
+
383
+ if debug:
384
+ print(
385
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
+ )
387
+
388
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
+
390
+
391
+ class GatheredTensor:
392
+ """
393
+ A pseudo tensor that collects partitioned weights.
394
+ It is more memory efficient when there are multiple groups.
395
+ """
396
+
397
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
+ self.flat_groups = flat_groups
399
+ self.flat_groups_offset = flat_groups_offset
400
+ self.offset = offset
401
+ self.partitioned_numel = partitioned_numel
402
+ self.shape = shape
403
+ self.dtype = self.flat_groups[0][0].dtype
404
+
405
+ def contiguous(self):
406
+ """
407
+ Merge partitioned weights from flat_groups into a single tensor.
408
+ """
409
+ end_idx = self.offset + self.partitioned_numel
410
+ world_size = len(self.flat_groups)
411
+ pad_flat_param_chunks = []
412
+
413
+ for rank_i in range(world_size):
414
+ # for each rank, we need to collect weights from related group/groups
415
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
416
+ start_group_id = None
417
+ end_group_id = None
418
+ for group_id in range(len(self.flat_groups_offset)):
419
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
+ start_group_id = group_id
421
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
+ end_group_id = group_id
423
+ break
424
+ # collect weights from related group/groups
425
+ for group_id in range(start_group_id, end_group_id + 1):
426
+ flat_tensor = flat_groups_at_rank_i[group_id]
427
+ start_offset = self.offset - self.flat_groups_offset[group_id]
428
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
+
431
+ # collect weights from all ranks
432
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
+ return param
435
+
436
+
437
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
+ param_shapes = zero_model_states[0].param_shapes
439
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
+
441
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
+ # param, re-consolidating each param, while dealing with padding if any
443
+
444
+ # merge list of dicts, preserving order
445
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
+
447
+ if debug:
448
+ for i in range(world_size):
449
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
+
451
+ wanted_params = len(param_shapes)
452
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
+ # not asserting if there is a mismatch due to possible padding
454
+ avail_numel = fp32_flat_groups[0].numel() * world_size
455
+ print(f"Trainable params: Have {avail_numel} numels to process.")
456
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
+
458
+ # params
459
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
+ # out-of-core computing solution
461
+ offset = 0
462
+ total_numel = 0
463
+ total_params = 0
464
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
+ unpartitioned_numel = shape.numel()
467
+ total_numel += unpartitioned_numel
468
+ total_params += 1
469
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
+
471
+ if debug:
472
+ print(
473
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
+ )
475
+
476
+ # memory efficient tensor
477
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
+ state_dict[name] = tensor
479
+ offset += partitioned_numel
480
+
481
+ offset *= world_size
482
+
483
+ # Sanity check
484
+ if offset != avail_numel:
485
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
+
487
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
+
489
+
490
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
+ exclude_frozen_parameters):
492
+ state_dict = OrderedDict()
493
+
494
+ # buffers
495
+ buffers = zero_model_states[0].buffers
496
+ state_dict.update(buffers)
497
+ if debug:
498
+ print(f"added {len(buffers)} buffers")
499
+
500
+ if not exclude_frozen_parameters:
501
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
+
503
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
+
505
+ # recover shared parameters
506
+ for pair in zero_model_states[0].shared_params:
507
+ if pair[1] in state_dict:
508
+ state_dict[pair[0]] = state_dict[pair[1]]
509
+
510
+ return state_dict
511
+
512
+
513
+ def to_torch_tensor(state_dict, return_empty_tensor=False):
514
+ """
515
+ Convert state_dict of GatheredTensor to torch tensor
516
+ """
517
+ torch_state_dict = {}
518
+ converted_tensors = {}
519
+ for name, tensor in state_dict.items():
520
+ tensor_id = id(tensor)
521
+ if tensor_id in converted_tensors: # shared tensors
522
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
+ torch_state_dict[name] = shared_tensor
524
+ else:
525
+ converted_tensors[tensor_id] = name
526
+ if return_empty_tensor:
527
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
+ else:
529
+ torch_state_dict[name] = tensor.contiguous()
530
+ return torch_state_dict
531
+
532
+
533
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
+ tag=None,
535
+ exclude_frozen_parameters=False,
536
+ lazy_mode=False):
537
+ """
538
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
+ via a model hub.
541
+
542
+ Args:
543
+ - ``checkpoint_dir``: path to the desired checkpoint folder
544
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
+ - ``exclude_frozen_parameters``: exclude frozen parameters
546
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
+
549
+ Returns:
550
+ - pytorch ``state_dict``
551
+
552
+ A typical usage might be ::
553
+
554
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
+ # do the training and checkpoint saving
556
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
+ model = model.cpu() # move to cpu
558
+ model.load_state_dict(state_dict)
559
+ # submit to model hub or save the model to share with others
560
+
561
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
+ application. i.e. you will need to re-initialize the deepspeed engine, since
563
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
+
565
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
+
567
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
+ the checkpoint. Or you can load state_dict in lazy mode ::
570
+
571
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
+ for name, lazy_tensor in state_dict.item():
574
+ tensor = lazy_tensor.contiguous() # to cpu
575
+ print(name, tensor)
576
+ # del tensor to release memory if it no longer in use
577
+ """
578
+ if tag is None:
579
+ latest_path = os.path.join(checkpoint_dir, 'latest')
580
+ if os.path.isfile(latest_path):
581
+ with open(latest_path, 'r') as fd:
582
+ tag = fd.read().strip()
583
+ else:
584
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
+
586
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
+
588
+ if not os.path.isdir(ds_checkpoint_dir):
589
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
+
591
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
+ if lazy_mode:
593
+ return state_dict
594
+ else:
595
+ return to_torch_tensor(state_dict)
596
+
597
+
598
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
+ output_dir,
600
+ max_shard_size="5GB",
601
+ safe_serialization=False,
602
+ tag=None,
603
+ exclude_frozen_parameters=False):
604
+ """
605
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
+
608
+ Args:
609
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
+ - ``exclude_frozen_parameters``: exclude frozen parameters
615
+ """
616
+
617
+ # Dependency pre-check
618
+ if safe_serialization:
619
+ try:
620
+ from safetensors.torch import save_file
621
+ except ImportError:
622
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
+ raise
624
+ if max_shard_size is not None:
625
+ try:
626
+ from huggingface_hub import split_torch_state_dict_into_shards
627
+ except ImportError:
628
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
+ raise
630
+
631
+ # Convert zero checkpoint to state_dict
632
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
+ tag,
634
+ exclude_frozen_parameters,
635
+ lazy_mode=True)
636
+
637
+ # Shard the model if it is too big.
638
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
+ if max_shard_size is not None:
640
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
+ # an memory-efficient approach for sharding
642
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
+ filename_pattern=filename_pattern,
645
+ max_shard_size=max_shard_size)
646
+ else:
647
+ from collections import namedtuple
648
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
+ state_dict_split = StateDictSplit(is_sharded=False,
650
+ filename_to_tensors={weights_name: list(state_dict.keys())})
651
+
652
+ # Save the model by shard
653
+ os.makedirs(output_dir, exist_ok=True)
654
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
+ shard_state_dict = to_torch_tensor(shard_state_dict)
658
+ output_path = os.path.join(output_dir, shard_file)
659
+ if safe_serialization:
660
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
+ else:
662
+ torch.save(shard_state_dict, output_path)
663
+ # release the memory of current shard
664
+ for tensor_name in list(shard_state_dict.keys()):
665
+ del state_dict[tensor_name]
666
+ del shard_state_dict[tensor_name]
667
+ del shard_state_dict
668
+ gc.collect()
669
+
670
+ # Save index if sharded
671
+ if state_dict_split.is_sharded:
672
+ index = {
673
+ "metadata": state_dict_split.metadata,
674
+ "weight_map": state_dict_split.tensor_to_filename,
675
+ }
676
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
+ save_index_file = os.path.join(output_dir, save_index_file)
678
+ with open(save_index_file, "w", encoding="utf-8") as f:
679
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
+ f.write(content)
681
+
682
+
683
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
+ """
685
+ 1. Put the provided model to cpu
686
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
+ 3. Load it into the provided model
688
+
689
+ Args:
690
+ - ``model``: the model object to update
691
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
+
694
+ Returns:
695
+ - ``model`: modified model
696
+
697
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
698
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
+ conveniently placed for you in the checkpoint folder.
700
+
701
+ A typical usage might be ::
702
+
703
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
+ # submit to model hub or save the model to share with others
706
+
707
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
+
711
+ """
712
+ logger.info("Extracting fp32 weights")
713
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
+
715
+ logger.info("Overwriting model with fp32 weights")
716
+ model = model.cpu()
717
+ model.load_state_dict(state_dict, strict=False)
718
+
719
+ return model
720
+
721
+
722
+ if __name__ == "__main__":
723
+ parser = argparse.ArgumentParser()
724
+ parser.add_argument("checkpoint_dir",
725
+ type=str,
726
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
+ parser.add_argument("output_dir",
728
+ type=str,
729
+ help="directory to the pytorch fp32 state_dict output files"
730
+ "(e.g. path/checkpoint-12-output/)")
731
+ parser.add_argument(
732
+ "--max_shard_size",
733
+ type=str,
734
+ default="5GB",
735
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
+ "without CPU OOM issues.")
739
+ parser.add_argument(
740
+ "--safe_serialization",
741
+ default=False,
742
+ action='store_true',
743
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
+ parser.add_argument("-t",
745
+ "--tag",
746
+ type=str,
747
+ default=None,
748
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
+ args = parser.parse_args()
752
+
753
+ debug = args.debug
754
+
755
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
+ args.output_dir,
757
+ max_shard_size=args.max_shard_size,
758
+ safe_serialization=args.safe_serialization,
759
+ tag=args.tag,
760
+ exclude_frozen_parameters=args.exclude_frozen_parameters)