diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..3b2de3a2b5b5521be8e5f82c8f79c621b78a6ede 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+merged/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..db87425a55564275e896a97157b7289fa63eb1d3
--- /dev/null
+++ b/README.md
@@ -0,0 +1,155 @@
+---
+license: other
+library_name: peft
+tags:
+- generated_from_trainer
+base_model: google/gemma-7b-it
+model-index:
+- name: out
+ results: []
+---
+
+
+
+[
](https://github.com/OpenAccess-AI-Collective/axolotl)
+See axolotl config
+
+axolotl version: `0.4.0`
+```yaml
+# use google/gemma-7b if you have access
+base_model: google/gemma-7b-it
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+
+load_in_8bit: false
+load_in_4bit: true
+strict: false
+
+# huggingface repo
+datasets:
+ - path: ./python-oasst/chunk_1.jsonl
+ type: oasst
+val_set_size: 0.1
+output_dir: ./out
+
+adapter: qlora
+lora_r: 32
+lora_alpha: 16
+lora_dropout: 0.05
+lora_target_linear: true
+
+sequence_len: 4096
+sample_packing: false
+pad_to_sequence_len: true
+
+wandb_project: gemma-7b-it
+wandb_entity:
+wandb_watch:
+wandb_name:
+wandb_log_model:
+
+
+gradient_accumulation_steps: 6
+micro_batch_size: 4
+num_epochs: 4
+optimizer: adamw_bnb_8bit
+lr_scheduler: cosine
+learning_rate: 0.0002
+
+train_on_inputs: true
+group_by_length: false
+bf16: auto
+fp16:
+tf32: false
+
+gradient_checkpointing: true
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+warmup_ratio: 0.1
+evals_per_epoch: 4
+eval_table_size:
+eval_max_new_tokens: 128
+saves_per_epoch: 1
+debug:
+deepspeed: deepspeed_configs/zero1.json
+weight_decay: 0.0
+fsdp:
+fsdp_config:
+special_tokens:
+
+```
+
+
+
+# out
+
+This model is a fine-tuned version of [google/gemma-7b-it](https://huggingface.co/google/gemma-7b-it) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.1911
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 6
+- total_train_batch_size: 96
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 9
+- num_epochs: 4
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 5.0474 | 0.01 | 1 | 5.9279 |
+| 1.2191 | 0.26 | 24 | 1.2947 |
+| 1.1165 | 0.51 | 48 | 1.1679 |
+| 1.0711 | 0.77 | 72 | 1.1377 |
+| 0.9546 | 1.02 | 96 | 1.1303 |
+| 0.9309 | 1.28 | 120 | 1.1298 |
+| 0.9588 | 1.54 | 144 | 1.1242 |
+| 0.8553 | 1.79 | 168 | 1.1259 |
+| 0.8231 | 2.05 | 192 | 1.1449 |
+| 0.8154 | 2.31 | 216 | 1.1514 |
+| 0.7354 | 2.56 | 240 | 1.1471 |
+| 0.7577 | 2.82 | 264 | 1.1479 |
+| 0.6647 | 3.07 | 288 | 1.1923 |
+| 0.6928 | 3.33 | 312 | 1.1856 |
+| 0.731 | 3.59 | 336 | 1.1890 |
+| 0.7193 | 3.84 | 360 | 1.1911 |
+
+
+### Framework versions
+
+- PEFT 0.9.0
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2+cu118
+- Datasets 2.18.0
+- Tokenizers 0.15.0
\ No newline at end of file
diff --git a/README1.md b/README1.md
new file mode 100644
index 0000000000000000000000000000000000000000..f81be350848f2e9d42a7b4d602590ea62a799d4d
--- /dev/null
+++ b/README1.md
@@ -0,0 +1,153 @@
+---
+license: other
+library_name: peft
+tags:
+- generated_from_trainer
+base_model: google/gemma-7b-it
+model-index:
+- name: out
+ results: []
+---
+
+
+[](https://github.com/OpenAccess-AI-Collective/axolotl)
+See axolotl config
+
+axolotl version: `0.4.0`
+```yaml
+# use google/gemma-7b if you have access
+base_model: google/gemma-7b-it
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+
+load_in_8bit: false
+load_in_4bit: true
+strict: false
+
+# huggingface repo
+datasets:
+ - path: ./python-oasst/chunk_1.jsonl
+ type: oasst
+val_set_size: 0.1
+output_dir: ./out
+
+adapter: qlora
+lora_r: 32
+lora_alpha: 16
+lora_dropout: 0.05
+lora_target_linear: true
+
+sequence_len: 4096
+sample_packing: false
+pad_to_sequence_len: true
+
+wandb_project: gemma-7b-it
+wandb_entity:
+wandb_watch:
+wandb_name:
+wandb_log_model:
+
+
+gradient_accumulation_steps: 6
+micro_batch_size: 4
+num_epochs: 4
+optimizer: adamw_bnb_8bit
+lr_scheduler: cosine
+learning_rate: 0.0002
+
+train_on_inputs: true
+group_by_length: false
+bf16: auto
+fp16:
+tf32: false
+
+gradient_checkpointing: true
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+warmup_ratio: 0.1
+evals_per_epoch: 4
+eval_table_size:
+eval_max_new_tokens: 128
+saves_per_epoch: 1
+debug:
+deepspeed: deepspeed_configs/zero1.json
+weight_decay: 0.0
+fsdp:
+fsdp_config:
+special_tokens:
+
+```
+
+
+
+# out
+
+This model is a fine-tuned version of [google/gemma-7b-it](https://huggingface.co/google/gemma-7b-it) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.1911
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 6
+- total_train_batch_size: 96
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 9
+- num_epochs: 4
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 5.0474 | 0.01 | 1 | 5.9279 |
+| 1.2191 | 0.26 | 24 | 1.2947 |
+| 1.1165 | 0.51 | 48 | 1.1679 |
+| 1.0711 | 0.77 | 72 | 1.1377 |
+| 0.9546 | 1.02 | 96 | 1.1303 |
+| 0.9309 | 1.28 | 120 | 1.1298 |
+| 0.9588 | 1.54 | 144 | 1.1242 |
+| 0.8553 | 1.79 | 168 | 1.1259 |
+| 0.8231 | 2.05 | 192 | 1.1449 |
+| 0.8154 | 2.31 | 216 | 1.1514 |
+| 0.7354 | 2.56 | 240 | 1.1471 |
+| 0.7577 | 2.82 | 264 | 1.1479 |
+| 0.6647 | 3.07 | 288 | 1.1923 |
+| 0.6928 | 3.33 | 312 | 1.1856 |
+| 0.731 | 3.59 | 336 | 1.1890 |
+| 0.7193 | 3.84 | 360 | 1.1911 |
+
+
+### Framework versions
+
+- PEFT 0.9.0
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2+cu118
+- Datasets 2.18.0
+- Tokenizers 0.15.0
diff --git a/adapter_config.json b/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fe237ab0e60126fdecc08e25cd5b3a8fd8dd6ac
--- /dev/null
+++ b/adapter_config.json
@@ -0,0 +1,33 @@
+{
+ "alpha_pattern": {},
+ "auto_mapping": null,
+ "base_model_name_or_path": "google/gemma-7b-it",
+ "bias": "none",
+ "fan_in_fan_out": null,
+ "inference_mode": false,
+ "init_lora_weights": true,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "gate_proj",
+ "up_proj",
+ "k_proj",
+ "q_proj",
+ "down_proj",
+ "v_proj",
+ "o_proj"
+ ],
+ "task_type": "CAUSAL_LM",
+ "use_dora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/adapter_model.bin b/adapter_model.bin
new file mode 100644
index 0000000000000000000000000000000000000000..9b16235254c7ff718229e03db0888fa9d6268c6f
--- /dev/null
+++ b/adapter_model.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25b9fe36c6655d98214b7521f5d8d9f662fc0c4007a06eeefea535ccfec3dc1e
+size 200078074
diff --git a/checkpoint-279/README.md b/checkpoint-279/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..cc48d049490be40c7b7654441aabee7b5bc360d2
--- /dev/null
+++ b/checkpoint-279/README.md
@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: google/gemma-7b-it
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.9.0
\ No newline at end of file
diff --git a/checkpoint-279/adapter_config.json b/checkpoint-279/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c3e2768efa37865b99706584c01866eafc6c45a
--- /dev/null
+++ b/checkpoint-279/adapter_config.json
@@ -0,0 +1,33 @@
+{
+ "alpha_pattern": {},
+ "auto_mapping": null,
+ "base_model_name_or_path": "google/gemma-7b-it",
+ "bias": "none",
+ "fan_in_fan_out": null,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "o_proj",
+ "k_proj",
+ "q_proj",
+ "gate_proj",
+ "up_proj",
+ "v_proj"
+ ],
+ "task_type": "CAUSAL_LM",
+ "use_dora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-279/adapter_model.safetensors b/checkpoint-279/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c3c04654ff450d67ec2b1b45e1d571ae0eab054a
--- /dev/null
+++ b/checkpoint-279/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0831f70d185dae9ca69f58be3eab596067ac52e75e3e97b46d23ecd486b83942
+size 200068904
diff --git a/checkpoint-279/global_step279/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-279/global_step279/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..21e633b4b77b4edfdd0f521632781b56cde0b080
--- /dev/null
+++ b/checkpoint-279/global_step279/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90f2db91b1ca035dfa781beb0567b7cfaaf6646de04cc9a82d8e80069e7a5b09
+size 150126608
diff --git a/checkpoint-279/global_step279/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-279/global_step279/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7b8a11bc59c7b492cae7464f459f5bbe16df132
--- /dev/null
+++ b/checkpoint-279/global_step279/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9ec41ba7f5c3131e00c854ec2bbfca98e6a3321e5f2ddf6efdc6056fa008c5a
+size 150126672
diff --git a/checkpoint-279/global_step279/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/checkpoint-279/global_step279/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae239de85d1d54e64dbe41942e1dc312125eec8d
--- /dev/null
+++ b/checkpoint-279/global_step279/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41f227ca1d3c19b4cd53567e28a2d395c2e804bd38dfd9bb3c937adab1daf5a3
+size 150126736
diff --git a/checkpoint-279/global_step279/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/checkpoint-279/global_step279/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..377fc6abb214d8490cc9e327ff988d2fb88c3b7d
--- /dev/null
+++ b/checkpoint-279/global_step279/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e486ddf3459c4f6befb004a9374e7e4fb9bd64bba72dd2e6f7051ee89939988
+size 150126736
diff --git a/checkpoint-279/global_step279/mp_rank_00_model_states.pt b/checkpoint-279/global_step279/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4cbf492a5e3f139b6bf68e9b70d9b07bcc7297e2
--- /dev/null
+++ b/checkpoint-279/global_step279/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:feed7b7a8694c54651374fb581d67d60790a016e23023446231557add62ffc80
+size 1896781286
diff --git a/checkpoint-279/latest b/checkpoint-279/latest
new file mode 100644
index 0000000000000000000000000000000000000000..c623f890b987e2811c5236bd9d38a5b12402d12f
--- /dev/null
+++ b/checkpoint-279/latest
@@ -0,0 +1 @@
+global_step279
\ No newline at end of file
diff --git a/checkpoint-279/rng_state_0.pth b/checkpoint-279/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..75b332cff3d885c7cfe4d0d4895a75d357d0ee57
--- /dev/null
+++ b/checkpoint-279/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a84c3f9fa55e23a5c4d93b108c705b57ba9a5ed816191e6dfbb6e72ad2857e6d
+size 15024
diff --git a/checkpoint-279/rng_state_1.pth b/checkpoint-279/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f6f6c8e2a014103a89d2b3d6d1487c3519b57de3
--- /dev/null
+++ b/checkpoint-279/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbb1da31ff41578c72556d0a8b9b94abf6be26bf16b6456ecd87d2b611f5b9bd
+size 15024
diff --git a/checkpoint-279/rng_state_2.pth b/checkpoint-279/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2f5371f3c4f4be42560e55b3bea0c6ad6f2770da
--- /dev/null
+++ b/checkpoint-279/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11a7b38529914886a43976df69af7f331315329e1d38788c57003ca4cd1a849f
+size 15024
diff --git a/checkpoint-279/rng_state_3.pth b/checkpoint-279/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..8a2020eda5d32507d342c00855009f3cd2cdc75e
--- /dev/null
+++ b/checkpoint-279/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d65b4248464f467db8226c5cc4ba4aa32e06af0bf915b61ea8a2db71d16b5ce
+size 15024
diff --git a/checkpoint-279/scheduler.pt b/checkpoint-279/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0110a56896fbbc605befb9fe2f14c329e7c4447
--- /dev/null
+++ b/checkpoint-279/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:099f524a0aa9353b01bf7d70e5a899c6e8ee8efc46e982213631888df6e5111b
+size 1064
diff --git a/checkpoint-279/trainer_state.json b/checkpoint-279/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..a035361cbd63e0415e26ba966a26d8dded996eba
--- /dev/null
+++ b/checkpoint-279/trainer_state.json
@@ -0,0 +1,2070 @@
+{
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 2.97864768683274,
+ "eval_steps": 24,
+ "global_step": 279,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.01,
+ "grad_norm": 1.8206765789002874,
+ "learning_rate": 2.2222222222222223e-05,
+ "loss": 5.0474,
+ "step": 1
+ },
+ {
+ "epoch": 0.01,
+ "eval_loss": 5.927858829498291,
+ "eval_runtime": 117.3665,
+ "eval_samples_per_second": 8.512,
+ "eval_steps_per_second": 0.537,
+ "step": 1
+ },
+ {
+ "epoch": 0.02,
+ "grad_norm": 1.9889295079554647,
+ "learning_rate": 4.4444444444444447e-05,
+ "loss": 5.5569,
+ "step": 2
+ },
+ {
+ "epoch": 0.03,
+ "grad_norm": 1.8931443004310682,
+ "learning_rate": 6.666666666666667e-05,
+ "loss": 5.2383,
+ "step": 3
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 2.195266234429632,
+ "learning_rate": 8.888888888888889e-05,
+ "loss": 5.4943,
+ "step": 4
+ },
+ {
+ "epoch": 0.05,
+ "grad_norm": 2.6001064132041503,
+ "learning_rate": 0.00011111111111111112,
+ "loss": 5.2602,
+ "step": 5
+ },
+ {
+ "epoch": 0.06,
+ "grad_norm": 3.26301463076567,
+ "learning_rate": 0.00013333333333333334,
+ "loss": 4.8182,
+ "step": 6
+ },
+ {
+ "epoch": 0.07,
+ "grad_norm": 3.476044691292363,
+ "learning_rate": 0.00015555555555555556,
+ "loss": 4.0432,
+ "step": 7
+ },
+ {
+ "epoch": 0.09,
+ "grad_norm": 3.378803229553045,
+ "learning_rate": 0.00017777777777777779,
+ "loss": 3.5212,
+ "step": 8
+ },
+ {
+ "epoch": 0.1,
+ "grad_norm": 3.9419449437137017,
+ "learning_rate": 0.0002,
+ "loss": 3.2239,
+ "step": 9
+ },
+ {
+ "epoch": 0.11,
+ "grad_norm": 5.8833082175146485,
+ "learning_rate": 0.00019999625498303932,
+ "loss": 3.4319,
+ "step": 10
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 5.4690223843996515,
+ "learning_rate": 0.0001999850202126604,
+ "loss": 2.8167,
+ "step": 11
+ },
+ {
+ "epoch": 0.13,
+ "grad_norm": 7.009614336449043,
+ "learning_rate": 0.00019996629653035126,
+ "loss": 2.7966,
+ "step": 12
+ },
+ {
+ "epoch": 0.14,
+ "grad_norm": 6.254841874500106,
+ "learning_rate": 0.0001999400853385221,
+ "loss": 2.1336,
+ "step": 13
+ },
+ {
+ "epoch": 0.15,
+ "grad_norm": 6.037710889841169,
+ "learning_rate": 0.00019990638860040006,
+ "loss": 1.85,
+ "step": 14
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0500019118881985,
+ "learning_rate": 0.00019986520883988232,
+ "loss": 1.5964,
+ "step": 15
+ },
+ {
+ "epoch": 0.17,
+ "grad_norm": 0.6169710624824223,
+ "learning_rate": 0.00019981654914134686,
+ "loss": 1.4307,
+ "step": 16
+ },
+ {
+ "epoch": 0.18,
+ "grad_norm": 1.86114059095932,
+ "learning_rate": 0.00019976041314942155,
+ "loss": 1.4285,
+ "step": 17
+ },
+ {
+ "epoch": 0.19,
+ "grad_norm": 1.6513877610200167,
+ "learning_rate": 0.00019969680506871137,
+ "loss": 1.4621,
+ "step": 18
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.4395882738454628,
+ "learning_rate": 0.000199625729663483,
+ "loss": 1.3561,
+ "step": 19
+ },
+ {
+ "epoch": 0.21,
+ "grad_norm": 0.70847060238536,
+ "learning_rate": 0.00019954719225730847,
+ "loss": 1.3565,
+ "step": 20
+ },
+ {
+ "epoch": 0.22,
+ "grad_norm": 0.4331630595385925,
+ "learning_rate": 0.00019946119873266613,
+ "loss": 1.3374,
+ "step": 21
+ },
+ {
+ "epoch": 0.23,
+ "grad_norm": 0.5580281682185451,
+ "learning_rate": 0.0001993677555305002,
+ "loss": 1.313,
+ "step": 22
+ },
+ {
+ "epoch": 0.25,
+ "grad_norm": 0.5217443953771937,
+ "learning_rate": 0.00019926686964973813,
+ "loss": 1.2541,
+ "step": 23
+ },
+ {
+ "epoch": 0.26,
+ "grad_norm": 0.36823120314463453,
+ "learning_rate": 0.00019915854864676664,
+ "loss": 1.2191,
+ "step": 24
+ },
+ {
+ "epoch": 0.26,
+ "eval_loss": 1.2946609258651733,
+ "eval_runtime": 118.9039,
+ "eval_samples_per_second": 8.402,
+ "eval_steps_per_second": 0.53,
+ "step": 24
+ },
+ {
+ "epoch": 0.27,
+ "grad_norm": 0.5797477063688413,
+ "learning_rate": 0.0001990428006348656,
+ "loss": 1.24,
+ "step": 25
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 0.41369538857234545,
+ "learning_rate": 0.00019891963428360043,
+ "loss": 1.209,
+ "step": 26
+ },
+ {
+ "epoch": 0.29,
+ "grad_norm": 0.36666008426797836,
+ "learning_rate": 0.00019878905881817252,
+ "loss": 1.2543,
+ "step": 27
+ },
+ {
+ "epoch": 0.3,
+ "grad_norm": 0.3976779691989045,
+ "learning_rate": 0.00019865108401872857,
+ "loss": 1.2431,
+ "step": 28
+ },
+ {
+ "epoch": 0.31,
+ "grad_norm": 0.4992861718630414,
+ "learning_rate": 0.00019850572021962788,
+ "loss": 1.2471,
+ "step": 29
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.33729072192890136,
+ "learning_rate": 0.00019835297830866826,
+ "loss": 1.1933,
+ "step": 30
+ },
+ {
+ "epoch": 0.33,
+ "grad_norm": 0.29373457949318904,
+ "learning_rate": 0.00019819286972627066,
+ "loss": 1.1761,
+ "step": 31
+ },
+ {
+ "epoch": 0.34,
+ "grad_norm": 0.5339184947140588,
+ "learning_rate": 0.0001980254064646223,
+ "loss": 1.165,
+ "step": 32
+ },
+ {
+ "epoch": 0.35,
+ "grad_norm": 0.38755069216510263,
+ "learning_rate": 0.00019785060106677818,
+ "loss": 1.1236,
+ "step": 33
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 0.338373181403367,
+ "learning_rate": 0.00019766846662572191,
+ "loss": 1.2102,
+ "step": 34
+ },
+ {
+ "epoch": 0.37,
+ "grad_norm": 0.39237714718744304,
+ "learning_rate": 0.00019747901678338496,
+ "loss": 1.1642,
+ "step": 35
+ },
+ {
+ "epoch": 0.38,
+ "grad_norm": 0.3614249847081747,
+ "learning_rate": 0.00019728226572962473,
+ "loss": 1.1387,
+ "step": 36
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.28278007479509987,
+ "learning_rate": 0.00019707822820116193,
+ "loss": 1.0939,
+ "step": 37
+ },
+ {
+ "epoch": 0.41,
+ "grad_norm": 0.3008254873268798,
+ "learning_rate": 0.00019686691948047664,
+ "loss": 1.1346,
+ "step": 38
+ },
+ {
+ "epoch": 0.42,
+ "grad_norm": 0.4263010439416343,
+ "learning_rate": 0.0001966483553946637,
+ "loss": 1.1015,
+ "step": 39
+ },
+ {
+ "epoch": 0.43,
+ "grad_norm": 0.32725448028464205,
+ "learning_rate": 0.00019642255231424729,
+ "loss": 1.1324,
+ "step": 40
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 0.3028242900588441,
+ "learning_rate": 0.00019618952715195475,
+ "loss": 1.1147,
+ "step": 41
+ },
+ {
+ "epoch": 0.45,
+ "grad_norm": 0.33893311928252234,
+ "learning_rate": 0.00019594929736144976,
+ "loss": 1.0978,
+ "step": 42
+ },
+ {
+ "epoch": 0.46,
+ "grad_norm": 0.2786082334492372,
+ "learning_rate": 0.0001957018809360251,
+ "loss": 1.0933,
+ "step": 43
+ },
+ {
+ "epoch": 0.47,
+ "grad_norm": 0.2732185168098956,
+ "learning_rate": 0.00019544729640725498,
+ "loss": 1.084,
+ "step": 44
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.33386436894143035,
+ "learning_rate": 0.00019518556284360696,
+ "loss": 1.0673,
+ "step": 45
+ },
+ {
+ "epoch": 0.49,
+ "grad_norm": 0.2761688734050621,
+ "learning_rate": 0.00019491669984901379,
+ "loss": 1.0523,
+ "step": 46
+ },
+ {
+ "epoch": 0.5,
+ "grad_norm": 0.3346957388610895,
+ "learning_rate": 0.00019464072756140486,
+ "loss": 1.0913,
+ "step": 47
+ },
+ {
+ "epoch": 0.51,
+ "grad_norm": 0.30196058996924285,
+ "learning_rate": 0.0001943576666511982,
+ "loss": 1.1165,
+ "step": 48
+ },
+ {
+ "epoch": 0.51,
+ "eval_loss": 1.167867660522461,
+ "eval_runtime": 119.1485,
+ "eval_samples_per_second": 8.384,
+ "eval_steps_per_second": 0.529,
+ "step": 48
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 0.27445390350987153,
+ "learning_rate": 0.00019406753831975203,
+ "loss": 1.1069,
+ "step": 49
+ },
+ {
+ "epoch": 0.53,
+ "grad_norm": 0.34729097228771255,
+ "learning_rate": 0.00019377036429777672,
+ "loss": 1.0567,
+ "step": 50
+ },
+ {
+ "epoch": 0.54,
+ "grad_norm": 0.31314016575739406,
+ "learning_rate": 0.0001934661668437073,
+ "loss": 1.0875,
+ "step": 51
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.29140014335226905,
+ "learning_rate": 0.0001931549687420364,
+ "loss": 1.0929,
+ "step": 52
+ },
+ {
+ "epoch": 0.57,
+ "grad_norm": 0.2638104110161505,
+ "learning_rate": 0.00019283679330160726,
+ "loss": 1.0963,
+ "step": 53
+ },
+ {
+ "epoch": 0.58,
+ "grad_norm": 0.2833945318119855,
+ "learning_rate": 0.0001925116643538684,
+ "loss": 1.0535,
+ "step": 54
+ },
+ {
+ "epoch": 0.59,
+ "grad_norm": 0.28672689795285417,
+ "learning_rate": 0.0001921796062510882,
+ "loss": 1.0699,
+ "step": 55
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 0.261255409262294,
+ "learning_rate": 0.00019184064386453128,
+ "loss": 1.0658,
+ "step": 56
+ },
+ {
+ "epoch": 0.61,
+ "grad_norm": 0.24304864434604007,
+ "learning_rate": 0.00019149480258259533,
+ "loss": 1.0441,
+ "step": 57
+ },
+ {
+ "epoch": 0.62,
+ "grad_norm": 0.2987107937915846,
+ "learning_rate": 0.00019114210830890969,
+ "loss": 1.0061,
+ "step": 58
+ },
+ {
+ "epoch": 0.63,
+ "grad_norm": 0.2617045441373282,
+ "learning_rate": 0.00019078258746039507,
+ "loss": 1.0578,
+ "step": 59
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.2577955355987167,
+ "learning_rate": 0.00019041626696528503,
+ "loss": 1.0333,
+ "step": 60
+ },
+ {
+ "epoch": 0.65,
+ "grad_norm": 0.2823058812174375,
+ "learning_rate": 0.0001900431742611089,
+ "loss": 1.0837,
+ "step": 61
+ },
+ {
+ "epoch": 0.66,
+ "grad_norm": 0.30425238718712166,
+ "learning_rate": 0.00018966333729263674,
+ "loss": 1.0619,
+ "step": 62
+ },
+ {
+ "epoch": 0.67,
+ "grad_norm": 0.29826831116146957,
+ "learning_rate": 0.0001892767845097864,
+ "loss": 1.056,
+ "step": 63
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 0.22990267950533677,
+ "learning_rate": 0.00018888354486549237,
+ "loss": 1.061,
+ "step": 64
+ },
+ {
+ "epoch": 0.69,
+ "grad_norm": 0.27604852373975236,
+ "learning_rate": 0.00018848364781353744,
+ "loss": 1.0624,
+ "step": 65
+ },
+ {
+ "epoch": 0.7,
+ "grad_norm": 0.302101014156969,
+ "learning_rate": 0.00018807712330634642,
+ "loss": 1.0965,
+ "step": 66
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.2532153192142023,
+ "learning_rate": 0.00018766400179274286,
+ "loss": 1.0972,
+ "step": 67
+ },
+ {
+ "epoch": 0.73,
+ "grad_norm": 0.23803088057755897,
+ "learning_rate": 0.00018724431421566823,
+ "loss": 1.0823,
+ "step": 68
+ },
+ {
+ "epoch": 0.74,
+ "grad_norm": 0.2200041903156331,
+ "learning_rate": 0.0001868180920098644,
+ "loss": 1.037,
+ "step": 69
+ },
+ {
+ "epoch": 0.75,
+ "grad_norm": 0.31123761066229655,
+ "learning_rate": 0.00018638536709951917,
+ "loss": 1.0689,
+ "step": 70
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 0.2760757149384919,
+ "learning_rate": 0.00018594617189587512,
+ "loss": 1.0071,
+ "step": 71
+ },
+ {
+ "epoch": 0.77,
+ "grad_norm": 0.2452672521810973,
+ "learning_rate": 0.00018550053929480202,
+ "loss": 1.0711,
+ "step": 72
+ },
+ {
+ "epoch": 0.77,
+ "eval_loss": 1.1377497911453247,
+ "eval_runtime": 119.461,
+ "eval_samples_per_second": 8.363,
+ "eval_steps_per_second": 0.527,
+ "step": 72
+ },
+ {
+ "epoch": 0.78,
+ "grad_norm": 0.30897216290479246,
+ "learning_rate": 0.0001850485026743328,
+ "loss": 1.0508,
+ "step": 73
+ },
+ {
+ "epoch": 0.79,
+ "grad_norm": 0.24165903393157925,
+ "learning_rate": 0.00018459009589216364,
+ "loss": 1.046,
+ "step": 74
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.2509819208307879,
+ "learning_rate": 0.00018412535328311814,
+ "loss": 1.0726,
+ "step": 75
+ },
+ {
+ "epoch": 0.81,
+ "grad_norm": 0.26145395006758515,
+ "learning_rate": 0.00018365430965657526,
+ "loss": 0.9998,
+ "step": 76
+ },
+ {
+ "epoch": 0.82,
+ "grad_norm": 0.26920709605794424,
+ "learning_rate": 0.00018317700029386245,
+ "loss": 1.065,
+ "step": 77
+ },
+ {
+ "epoch": 0.83,
+ "grad_norm": 0.24226754926786417,
+ "learning_rate": 0.0001826934609456129,
+ "loss": 1.0489,
+ "step": 78
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 0.3022365661006827,
+ "learning_rate": 0.00018220372782908777,
+ "loss": 1.0372,
+ "step": 79
+ },
+ {
+ "epoch": 0.85,
+ "grad_norm": 0.25795710005352673,
+ "learning_rate": 0.00018170783762546365,
+ "loss": 1.0128,
+ "step": 80
+ },
+ {
+ "epoch": 0.86,
+ "grad_norm": 0.3490748875058354,
+ "learning_rate": 0.00018120582747708502,
+ "loss": 1.0168,
+ "step": 81
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.24938209735120945,
+ "learning_rate": 0.00018069773498468223,
+ "loss": 0.9586,
+ "step": 82
+ },
+ {
+ "epoch": 0.89,
+ "grad_norm": 0.2527612545099894,
+ "learning_rate": 0.00018018359820455536,
+ "loss": 1.0385,
+ "step": 83
+ },
+ {
+ "epoch": 0.9,
+ "grad_norm": 0.27528879975094916,
+ "learning_rate": 0.0001796634556457236,
+ "loss": 1.0328,
+ "step": 84
+ },
+ {
+ "epoch": 0.91,
+ "grad_norm": 0.2605002777661913,
+ "learning_rate": 0.0001791373462670411,
+ "loss": 0.9966,
+ "step": 85
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 0.3117107796665858,
+ "learning_rate": 0.00017860530947427875,
+ "loss": 0.9772,
+ "step": 86
+ },
+ {
+ "epoch": 0.93,
+ "grad_norm": 0.28336227154677734,
+ "learning_rate": 0.0001780673851171728,
+ "loss": 1.0724,
+ "step": 87
+ },
+ {
+ "epoch": 0.94,
+ "grad_norm": 0.42707817919652674,
+ "learning_rate": 0.0001775236134864401,
+ "loss": 1.0038,
+ "step": 88
+ },
+ {
+ "epoch": 0.95,
+ "grad_norm": 0.29236016959846456,
+ "learning_rate": 0.0001769740353107602,
+ "loss": 1.0083,
+ "step": 89
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 0.43295063403530637,
+ "learning_rate": 0.00017641869175372493,
+ "loss": 1.022,
+ "step": 90
+ },
+ {
+ "epoch": 0.97,
+ "grad_norm": 0.3086663897043129,
+ "learning_rate": 0.00017585762441075503,
+ "loss": 1.0303,
+ "step": 91
+ },
+ {
+ "epoch": 0.98,
+ "grad_norm": 0.2783768981163154,
+ "learning_rate": 0.0001752908753059849,
+ "loss": 1.061,
+ "step": 92
+ },
+ {
+ "epoch": 0.99,
+ "grad_norm": 0.43168501819843275,
+ "learning_rate": 0.00017471848688911464,
+ "loss": 1.0631,
+ "step": 93
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 0.25487494913299935,
+ "learning_rate": 0.0001741405020322309,
+ "loss": 0.9858,
+ "step": 94
+ },
+ {
+ "epoch": 1.01,
+ "grad_norm": 0.3229761094582219,
+ "learning_rate": 0.00017355696402659548,
+ "loss": 0.9495,
+ "step": 95
+ },
+ {
+ "epoch": 1.02,
+ "grad_norm": 0.3178464701266748,
+ "learning_rate": 0.000172967916579403,
+ "loss": 0.9546,
+ "step": 96
+ },
+ {
+ "epoch": 1.02,
+ "eval_loss": 1.1303094625473022,
+ "eval_runtime": 119.6761,
+ "eval_samples_per_second": 8.348,
+ "eval_steps_per_second": 0.526,
+ "step": 96
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.2534616980189548,
+ "learning_rate": 0.00017237340381050703,
+ "loss": 0.9509,
+ "step": 97
+ },
+ {
+ "epoch": 1.05,
+ "grad_norm": 0.2354382873554396,
+ "learning_rate": 0.00017177347024911562,
+ "loss": 0.9611,
+ "step": 98
+ },
+ {
+ "epoch": 1.06,
+ "grad_norm": 0.2754259154521738,
+ "learning_rate": 0.00017116816083045602,
+ "loss": 0.9184,
+ "step": 99
+ },
+ {
+ "epoch": 1.07,
+ "grad_norm": 0.25868181129480755,
+ "learning_rate": 0.00017055752089240907,
+ "loss": 0.957,
+ "step": 100
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 0.2383943586330267,
+ "learning_rate": 0.00016994159617211317,
+ "loss": 0.9638,
+ "step": 101
+ },
+ {
+ "epoch": 1.09,
+ "grad_norm": 0.2706420372628291,
+ "learning_rate": 0.0001693204328025389,
+ "loss": 0.9115,
+ "step": 102
+ },
+ {
+ "epoch": 1.1,
+ "grad_norm": 0.2751042656041904,
+ "learning_rate": 0.0001686940773090333,
+ "loss": 0.9277,
+ "step": 103
+ },
+ {
+ "epoch": 1.11,
+ "grad_norm": 0.27700872737428867,
+ "learning_rate": 0.00016806257660583534,
+ "loss": 0.9248,
+ "step": 104
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.3350046312844708,
+ "learning_rate": 0.00016742597799256182,
+ "loss": 0.928,
+ "step": 105
+ },
+ {
+ "epoch": 1.13,
+ "grad_norm": 0.4055944986440079,
+ "learning_rate": 0.00016678432915066488,
+ "loss": 0.9074,
+ "step": 106
+ },
+ {
+ "epoch": 1.14,
+ "grad_norm": 0.2515177402600531,
+ "learning_rate": 0.00016613767813986044,
+ "loss": 0.9564,
+ "step": 107
+ },
+ {
+ "epoch": 1.15,
+ "grad_norm": 0.2571149695502646,
+ "learning_rate": 0.00016548607339452853,
+ "loss": 0.93,
+ "step": 108
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 0.38608942941048996,
+ "learning_rate": 0.0001648295637200856,
+ "loss": 0.9281,
+ "step": 109
+ },
+ {
+ "epoch": 1.17,
+ "grad_norm": 0.31939838976976676,
+ "learning_rate": 0.000164168198289329,
+ "loss": 0.9914,
+ "step": 110
+ },
+ {
+ "epoch": 1.19,
+ "grad_norm": 0.30504937567650897,
+ "learning_rate": 0.00016350202663875386,
+ "loss": 0.9549,
+ "step": 111
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.3320388344291162,
+ "learning_rate": 0.0001628310986648427,
+ "loss": 0.9086,
+ "step": 112
+ },
+ {
+ "epoch": 1.21,
+ "grad_norm": 0.27715569151296165,
+ "learning_rate": 0.0001621554646203284,
+ "loss": 0.8537,
+ "step": 113
+ },
+ {
+ "epoch": 1.22,
+ "grad_norm": 0.278787508566418,
+ "learning_rate": 0.0001614751751104301,
+ "loss": 0.9354,
+ "step": 114
+ },
+ {
+ "epoch": 1.23,
+ "grad_norm": 0.24483614460003267,
+ "learning_rate": 0.00016079028108906282,
+ "loss": 0.8996,
+ "step": 115
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 0.37520609596400134,
+ "learning_rate": 0.0001601008338550211,
+ "loss": 0.9514,
+ "step": 116
+ },
+ {
+ "epoch": 1.25,
+ "grad_norm": 0.2565631505653599,
+ "learning_rate": 0.00015940688504813662,
+ "loss": 0.8984,
+ "step": 117
+ },
+ {
+ "epoch": 1.26,
+ "grad_norm": 0.26348552476529935,
+ "learning_rate": 0.00015870848664541044,
+ "loss": 0.8941,
+ "step": 118
+ },
+ {
+ "epoch": 1.27,
+ "grad_norm": 0.32431198985496534,
+ "learning_rate": 0.00015800569095711982,
+ "loss": 0.8876,
+ "step": 119
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 0.29308039763069227,
+ "learning_rate": 0.00015729855062290022,
+ "loss": 0.9309,
+ "step": 120
+ },
+ {
+ "epoch": 1.28,
+ "eval_loss": 1.129751205444336,
+ "eval_runtime": 119.1497,
+ "eval_samples_per_second": 8.384,
+ "eval_steps_per_second": 0.529,
+ "step": 120
+ },
+ {
+ "epoch": 1.29,
+ "grad_norm": 0.2793291380060977,
+ "learning_rate": 0.0001565871186078025,
+ "loss": 0.9453,
+ "step": 121
+ },
+ {
+ "epoch": 1.3,
+ "grad_norm": 0.28873644301555734,
+ "learning_rate": 0.000155871448198326,
+ "loss": 0.9243,
+ "step": 122
+ },
+ {
+ "epoch": 1.31,
+ "grad_norm": 0.3086103724578039,
+ "learning_rate": 0.00015515159299842707,
+ "loss": 0.8877,
+ "step": 123
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 0.30407892484693505,
+ "learning_rate": 0.00015442760692550443,
+ "loss": 0.9448,
+ "step": 124
+ },
+ {
+ "epoch": 1.33,
+ "grad_norm": 0.29771602861368474,
+ "learning_rate": 0.00015369954420636048,
+ "loss": 0.889,
+ "step": 125
+ },
+ {
+ "epoch": 1.35,
+ "grad_norm": 0.30480490158838136,
+ "learning_rate": 0.00015296745937313987,
+ "loss": 0.9405,
+ "step": 126
+ },
+ {
+ "epoch": 1.36,
+ "grad_norm": 0.2949192855418127,
+ "learning_rate": 0.00015223140725924495,
+ "loss": 0.9382,
+ "step": 127
+ },
+ {
+ "epoch": 1.37,
+ "grad_norm": 0.2813631863132807,
+ "learning_rate": 0.00015149144299522873,
+ "loss": 0.9526,
+ "step": 128
+ },
+ {
+ "epoch": 1.38,
+ "grad_norm": 0.28548924064070513,
+ "learning_rate": 0.00015074762200466556,
+ "loss": 0.9174,
+ "step": 129
+ },
+ {
+ "epoch": 1.39,
+ "grad_norm": 0.28137053449960464,
+ "learning_rate": 0.00015000000000000001,
+ "loss": 0.9244,
+ "step": 130
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 0.2626750895717777,
+ "learning_rate": 0.00014924863297837378,
+ "loss": 0.9335,
+ "step": 131
+ },
+ {
+ "epoch": 1.41,
+ "grad_norm": 0.26686502371015536,
+ "learning_rate": 0.00014849357721743168,
+ "loss": 0.8948,
+ "step": 132
+ },
+ {
+ "epoch": 1.42,
+ "grad_norm": 0.3332273481179679,
+ "learning_rate": 0.00014773488927110633,
+ "loss": 0.9274,
+ "step": 133
+ },
+ {
+ "epoch": 1.43,
+ "grad_norm": 0.2528048763375234,
+ "learning_rate": 0.00014697262596538227,
+ "loss": 0.8731,
+ "step": 134
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.27184211707488076,
+ "learning_rate": 0.00014620684439403962,
+ "loss": 0.9318,
+ "step": 135
+ },
+ {
+ "epoch": 1.45,
+ "grad_norm": 0.3051111137538683,
+ "learning_rate": 0.0001454376019143779,
+ "loss": 0.9447,
+ "step": 136
+ },
+ {
+ "epoch": 1.46,
+ "grad_norm": 0.28771401659835155,
+ "learning_rate": 0.00014466495614291977,
+ "loss": 0.9343,
+ "step": 137
+ },
+ {
+ "epoch": 1.47,
+ "grad_norm": 0.28995797921621524,
+ "learning_rate": 0.0001438889649510956,
+ "loss": 0.8978,
+ "step": 138
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 0.2749930548874636,
+ "learning_rate": 0.00014310968646090883,
+ "loss": 0.924,
+ "step": 139
+ },
+ {
+ "epoch": 1.49,
+ "grad_norm": 0.3097189537380989,
+ "learning_rate": 0.0001423271790405828,
+ "loss": 0.9574,
+ "step": 140
+ },
+ {
+ "epoch": 1.51,
+ "grad_norm": 0.2449218990319832,
+ "learning_rate": 0.00014154150130018866,
+ "loss": 0.8475,
+ "step": 141
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 0.24856388098419674,
+ "learning_rate": 0.0001407527120872557,
+ "loss": 0.9381,
+ "step": 142
+ },
+ {
+ "epoch": 1.53,
+ "grad_norm": 0.3169861882853132,
+ "learning_rate": 0.00013996087048236358,
+ "loss": 0.9141,
+ "step": 143
+ },
+ {
+ "epoch": 1.54,
+ "grad_norm": 0.30689184261103974,
+ "learning_rate": 0.00013916603579471705,
+ "loss": 0.9588,
+ "step": 144
+ },
+ {
+ "epoch": 1.54,
+ "eval_loss": 1.1242448091506958,
+ "eval_runtime": 119.0725,
+ "eval_samples_per_second": 8.39,
+ "eval_steps_per_second": 0.529,
+ "step": 144
+ },
+ {
+ "epoch": 1.55,
+ "grad_norm": 0.2961514212977567,
+ "learning_rate": 0.00013836826755770384,
+ "loss": 0.9371,
+ "step": 145
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 0.30790856503439346,
+ "learning_rate": 0.00013756762552443553,
+ "loss": 0.9612,
+ "step": 146
+ },
+ {
+ "epoch": 1.57,
+ "grad_norm": 0.3517398492864053,
+ "learning_rate": 0.000136764169663272,
+ "loss": 0.9253,
+ "step": 147
+ },
+ {
+ "epoch": 1.58,
+ "grad_norm": 0.26375798832515857,
+ "learning_rate": 0.00013595796015332984,
+ "loss": 0.8977,
+ "step": 148
+ },
+ {
+ "epoch": 1.59,
+ "grad_norm": 0.274348892672977,
+ "learning_rate": 0.00013514905737997473,
+ "loss": 0.8817,
+ "step": 149
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.35917564750751624,
+ "learning_rate": 0.00013433752193029886,
+ "loss": 0.886,
+ "step": 150
+ },
+ {
+ "epoch": 1.61,
+ "grad_norm": 0.38175124377914293,
+ "learning_rate": 0.00013352341458858265,
+ "loss": 0.8576,
+ "step": 151
+ },
+ {
+ "epoch": 1.62,
+ "grad_norm": 0.249633953215678,
+ "learning_rate": 0.00013270679633174218,
+ "loss": 1.0066,
+ "step": 152
+ },
+ {
+ "epoch": 1.63,
+ "grad_norm": 0.33494494430574784,
+ "learning_rate": 0.00013188772832476188,
+ "loss": 0.884,
+ "step": 153
+ },
+ {
+ "epoch": 1.64,
+ "grad_norm": 0.4176467296744032,
+ "learning_rate": 0.00013106627191611332,
+ "loss": 0.9041,
+ "step": 154
+ },
+ {
+ "epoch": 1.65,
+ "grad_norm": 0.27051479454532207,
+ "learning_rate": 0.00013024248863316012,
+ "loss": 0.8764,
+ "step": 155
+ },
+ {
+ "epoch": 1.67,
+ "grad_norm": 0.29302599029848847,
+ "learning_rate": 0.00012941644017754964,
+ "loss": 0.9786,
+ "step": 156
+ },
+ {
+ "epoch": 1.68,
+ "grad_norm": 0.3127378512248151,
+ "learning_rate": 0.00012858818842059145,
+ "loss": 0.9176,
+ "step": 157
+ },
+ {
+ "epoch": 1.69,
+ "grad_norm": 0.40647077063662906,
+ "learning_rate": 0.00012775779539862304,
+ "loss": 0.9387,
+ "step": 158
+ },
+ {
+ "epoch": 1.7,
+ "grad_norm": 0.29290601694481777,
+ "learning_rate": 0.00012692532330836346,
+ "loss": 0.9192,
+ "step": 159
+ },
+ {
+ "epoch": 1.71,
+ "grad_norm": 0.2819168741245354,
+ "learning_rate": 0.0001260908345022547,
+ "loss": 0.9253,
+ "step": 160
+ },
+ {
+ "epoch": 1.72,
+ "grad_norm": 0.3772714091394927,
+ "learning_rate": 0.00012525439148379128,
+ "loss": 0.9264,
+ "step": 161
+ },
+ {
+ "epoch": 1.73,
+ "grad_norm": 0.29399851067321503,
+ "learning_rate": 0.00012441605690283915,
+ "loss": 0.9357,
+ "step": 162
+ },
+ {
+ "epoch": 1.74,
+ "grad_norm": 0.2623180246832513,
+ "learning_rate": 0.00012357589355094275,
+ "loss": 0.8516,
+ "step": 163
+ },
+ {
+ "epoch": 1.75,
+ "grad_norm": 0.27796942024085824,
+ "learning_rate": 0.00012273396435662212,
+ "loss": 0.9328,
+ "step": 164
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 0.3107670297529076,
+ "learning_rate": 0.0001218903323806595,
+ "loss": 0.8769,
+ "step": 165
+ },
+ {
+ "epoch": 1.77,
+ "grad_norm": 0.2865573350738354,
+ "learning_rate": 0.00012104506081137608,
+ "loss": 0.9015,
+ "step": 166
+ },
+ {
+ "epoch": 1.78,
+ "grad_norm": 0.30595087117636693,
+ "learning_rate": 0.00012019821295989912,
+ "loss": 0.94,
+ "step": 167
+ },
+ {
+ "epoch": 1.79,
+ "grad_norm": 0.32540365653257874,
+ "learning_rate": 0.00011934985225541998,
+ "loss": 0.8553,
+ "step": 168
+ },
+ {
+ "epoch": 1.79,
+ "eval_loss": 1.1259374618530273,
+ "eval_runtime": 119.4351,
+ "eval_samples_per_second": 8.364,
+ "eval_steps_per_second": 0.527,
+ "step": 168
+ },
+ {
+ "epoch": 1.8,
+ "grad_norm": 0.3058868303314457,
+ "learning_rate": 0.00011850004224044315,
+ "loss": 0.9074,
+ "step": 169
+ },
+ {
+ "epoch": 1.81,
+ "grad_norm": 0.33266760488242775,
+ "learning_rate": 0.0001176488465660271,
+ "loss": 0.8799,
+ "step": 170
+ },
+ {
+ "epoch": 1.83,
+ "grad_norm": 0.3101183375673487,
+ "learning_rate": 0.00011679632898701649,
+ "loss": 0.9004,
+ "step": 171
+ },
+ {
+ "epoch": 1.84,
+ "grad_norm": 0.31535579418195775,
+ "learning_rate": 0.00011594255335726724,
+ "loss": 0.9238,
+ "step": 172
+ },
+ {
+ "epoch": 1.85,
+ "grad_norm": 0.28341827112854334,
+ "learning_rate": 0.00011508758362486358,
+ "loss": 0.9138,
+ "step": 173
+ },
+ {
+ "epoch": 1.86,
+ "grad_norm": 0.25699888796695625,
+ "learning_rate": 0.00011423148382732853,
+ "loss": 0.9175,
+ "step": 174
+ },
+ {
+ "epoch": 1.87,
+ "grad_norm": 0.29504332662698246,
+ "learning_rate": 0.0001133743180868273,
+ "loss": 0.9023,
+ "step": 175
+ },
+ {
+ "epoch": 1.88,
+ "grad_norm": 0.2993175263873948,
+ "learning_rate": 0.0001125161506053646,
+ "loss": 0.8893,
+ "step": 176
+ },
+ {
+ "epoch": 1.89,
+ "grad_norm": 0.2762659379409218,
+ "learning_rate": 0.00011165704565997593,
+ "loss": 0.9071,
+ "step": 177
+ },
+ {
+ "epoch": 1.9,
+ "grad_norm": 0.23620994229530515,
+ "learning_rate": 0.00011079706759791311,
+ "loss": 0.8796,
+ "step": 178
+ },
+ {
+ "epoch": 1.91,
+ "grad_norm": 0.28317619721877,
+ "learning_rate": 0.00010993628083182467,
+ "loss": 0.8983,
+ "step": 179
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 0.3252854551640304,
+ "learning_rate": 0.00010907474983493144,
+ "loss": 0.8947,
+ "step": 180
+ },
+ {
+ "epoch": 1.93,
+ "grad_norm": 0.2579136274422669,
+ "learning_rate": 0.00010821253913619726,
+ "loss": 0.8726,
+ "step": 181
+ },
+ {
+ "epoch": 1.94,
+ "grad_norm": 0.27201912720918364,
+ "learning_rate": 0.00010734971331549603,
+ "loss": 0.891,
+ "step": 182
+ },
+ {
+ "epoch": 1.95,
+ "grad_norm": 0.41257277193589503,
+ "learning_rate": 0.0001064863369987743,
+ "loss": 0.9188,
+ "step": 183
+ },
+ {
+ "epoch": 1.96,
+ "grad_norm": 0.264920112831242,
+ "learning_rate": 0.00010562247485321115,
+ "loss": 0.8761,
+ "step": 184
+ },
+ {
+ "epoch": 1.98,
+ "grad_norm": 0.28166441056422037,
+ "learning_rate": 0.00010475819158237425,
+ "loss": 0.8805,
+ "step": 185
+ },
+ {
+ "epoch": 1.99,
+ "grad_norm": 0.2818961139392159,
+ "learning_rate": 0.00010389355192137377,
+ "loss": 0.8934,
+ "step": 186
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 0.27424787600345923,
+ "learning_rate": 0.00010302862063201367,
+ "loss": 0.9237,
+ "step": 187
+ },
+ {
+ "epoch": 2.01,
+ "grad_norm": 0.25570082666079225,
+ "learning_rate": 0.00010216346249794087,
+ "loss": 0.8656,
+ "step": 188
+ },
+ {
+ "epoch": 2.02,
+ "grad_norm": 0.2712359904481713,
+ "learning_rate": 0.0001012981423197931,
+ "loss": 0.7627,
+ "step": 189
+ },
+ {
+ "epoch": 2.03,
+ "grad_norm": 0.25054404547068676,
+ "learning_rate": 0.00010043272491034523,
+ "loss": 0.8142,
+ "step": 190
+ },
+ {
+ "epoch": 2.04,
+ "grad_norm": 0.28520868420260026,
+ "learning_rate": 9.956727508965481e-05,
+ "loss": 0.7953,
+ "step": 191
+ },
+ {
+ "epoch": 2.05,
+ "grad_norm": 0.29413880984694873,
+ "learning_rate": 9.870185768020693e-05,
+ "loss": 0.8231,
+ "step": 192
+ },
+ {
+ "epoch": 2.05,
+ "eval_loss": 1.144862413406372,
+ "eval_runtime": 119.3004,
+ "eval_samples_per_second": 8.374,
+ "eval_steps_per_second": 0.528,
+ "step": 192
+ },
+ {
+ "epoch": 2.06,
+ "grad_norm": 0.28378300985247035,
+ "learning_rate": 9.783653750205915e-05,
+ "loss": 0.7478,
+ "step": 193
+ },
+ {
+ "epoch": 2.07,
+ "grad_norm": 0.31792721348179676,
+ "learning_rate": 9.697137936798634e-05,
+ "loss": 0.7961,
+ "step": 194
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 0.3291666436295964,
+ "learning_rate": 9.610644807862625e-05,
+ "loss": 0.7434,
+ "step": 195
+ },
+ {
+ "epoch": 2.09,
+ "grad_norm": 0.301579259001567,
+ "learning_rate": 9.524180841762577e-05,
+ "loss": 0.7779,
+ "step": 196
+ },
+ {
+ "epoch": 2.1,
+ "grad_norm": 0.30252161240414444,
+ "learning_rate": 9.437752514678887e-05,
+ "loss": 0.7689,
+ "step": 197
+ },
+ {
+ "epoch": 2.11,
+ "grad_norm": 0.3350657085129171,
+ "learning_rate": 9.35136630012257e-05,
+ "loss": 0.7574,
+ "step": 198
+ },
+ {
+ "epoch": 2.12,
+ "grad_norm": 0.3053109929956358,
+ "learning_rate": 9.265028668450402e-05,
+ "loss": 0.7729,
+ "step": 199
+ },
+ {
+ "epoch": 2.14,
+ "grad_norm": 0.30367223609567207,
+ "learning_rate": 9.178746086380275e-05,
+ "loss": 0.8111,
+ "step": 200
+ },
+ {
+ "epoch": 2.15,
+ "grad_norm": 0.3366440949136126,
+ "learning_rate": 9.092525016506858e-05,
+ "loss": 0.7986,
+ "step": 201
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 0.3228036608413652,
+ "learning_rate": 9.006371916817534e-05,
+ "loss": 0.8382,
+ "step": 202
+ },
+ {
+ "epoch": 2.17,
+ "grad_norm": 0.2919040789403488,
+ "learning_rate": 8.920293240208694e-05,
+ "loss": 0.7696,
+ "step": 203
+ },
+ {
+ "epoch": 2.18,
+ "grad_norm": 0.30084198177583166,
+ "learning_rate": 8.83429543400241e-05,
+ "loss": 0.7671,
+ "step": 204
+ },
+ {
+ "epoch": 2.19,
+ "grad_norm": 0.33931609000743107,
+ "learning_rate": 8.748384939463543e-05,
+ "loss": 0.7553,
+ "step": 205
+ },
+ {
+ "epoch": 2.2,
+ "grad_norm": 0.30413284924824485,
+ "learning_rate": 8.662568191317273e-05,
+ "loss": 0.7324,
+ "step": 206
+ },
+ {
+ "epoch": 2.21,
+ "grad_norm": 0.3014038998090481,
+ "learning_rate": 8.57685161726715e-05,
+ "loss": 0.7567,
+ "step": 207
+ },
+ {
+ "epoch": 2.22,
+ "grad_norm": 0.3176466329519527,
+ "learning_rate": 8.491241637513644e-05,
+ "loss": 0.8222,
+ "step": 208
+ },
+ {
+ "epoch": 2.23,
+ "grad_norm": 0.29981213041628285,
+ "learning_rate": 8.405744664273278e-05,
+ "loss": 0.7077,
+ "step": 209
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 0.2937916452228122,
+ "learning_rate": 8.320367101298351e-05,
+ "loss": 0.7231,
+ "step": 210
+ },
+ {
+ "epoch": 2.25,
+ "grad_norm": 0.32040684171320816,
+ "learning_rate": 8.235115343397295e-05,
+ "loss": 0.7556,
+ "step": 211
+ },
+ {
+ "epoch": 2.26,
+ "grad_norm": 0.31083028085316033,
+ "learning_rate": 8.149995775955686e-05,
+ "loss": 0.7514,
+ "step": 212
+ },
+ {
+ "epoch": 2.27,
+ "grad_norm": 0.3215465383581194,
+ "learning_rate": 8.065014774458003e-05,
+ "loss": 0.7933,
+ "step": 213
+ },
+ {
+ "epoch": 2.28,
+ "grad_norm": 0.3081200259196015,
+ "learning_rate": 7.980178704010089e-05,
+ "loss": 0.8062,
+ "step": 214
+ },
+ {
+ "epoch": 2.3,
+ "grad_norm": 0.3333248296288759,
+ "learning_rate": 7.895493918862396e-05,
+ "loss": 0.7784,
+ "step": 215
+ },
+ {
+ "epoch": 2.31,
+ "grad_norm": 0.3301326097292383,
+ "learning_rate": 7.810966761934053e-05,
+ "loss": 0.8154,
+ "step": 216
+ },
+ {
+ "epoch": 2.31,
+ "eval_loss": 1.1513652801513672,
+ "eval_runtime": 119.4371,
+ "eval_samples_per_second": 8.364,
+ "eval_steps_per_second": 0.527,
+ "step": 216
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 0.3166760836422428,
+ "learning_rate": 7.726603564337791e-05,
+ "loss": 0.7486,
+ "step": 217
+ },
+ {
+ "epoch": 2.33,
+ "grad_norm": 0.31309757318131876,
+ "learning_rate": 7.642410644905726e-05,
+ "loss": 0.771,
+ "step": 218
+ },
+ {
+ "epoch": 2.34,
+ "grad_norm": 0.36968796131043985,
+ "learning_rate": 7.558394309716088e-05,
+ "loss": 0.8051,
+ "step": 219
+ },
+ {
+ "epoch": 2.35,
+ "grad_norm": 0.27537675917328025,
+ "learning_rate": 7.474560851620873e-05,
+ "loss": 0.7536,
+ "step": 220
+ },
+ {
+ "epoch": 2.36,
+ "grad_norm": 0.2878011945022053,
+ "learning_rate": 7.390916549774536e-05,
+ "loss": 0.8126,
+ "step": 221
+ },
+ {
+ "epoch": 2.37,
+ "grad_norm": 0.3172405217395398,
+ "learning_rate": 7.307467669163655e-05,
+ "loss": 0.8156,
+ "step": 222
+ },
+ {
+ "epoch": 2.38,
+ "grad_norm": 0.3183651086957915,
+ "learning_rate": 7.224220460137701e-05,
+ "loss": 0.7821,
+ "step": 223
+ },
+ {
+ "epoch": 2.39,
+ "grad_norm": 0.3318078467573977,
+ "learning_rate": 7.141181157940859e-05,
+ "loss": 0.7993,
+ "step": 224
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 0.28446170407344085,
+ "learning_rate": 7.058355982245037e-05,
+ "loss": 0.7987,
+ "step": 225
+ },
+ {
+ "epoch": 2.41,
+ "grad_norm": 0.33568352702219995,
+ "learning_rate": 6.97575113668399e-05,
+ "loss": 0.773,
+ "step": 226
+ },
+ {
+ "epoch": 2.42,
+ "grad_norm": 0.30820575901544944,
+ "learning_rate": 6.893372808388675e-05,
+ "loss": 0.813,
+ "step": 227
+ },
+ {
+ "epoch": 2.43,
+ "grad_norm": 0.3121364386024255,
+ "learning_rate": 6.811227167523815e-05,
+ "loss": 0.7716,
+ "step": 228
+ },
+ {
+ "epoch": 2.44,
+ "grad_norm": 0.3211455560922844,
+ "learning_rate": 6.729320366825784e-05,
+ "loss": 0.7577,
+ "step": 229
+ },
+ {
+ "epoch": 2.46,
+ "grad_norm": 0.3315601260165869,
+ "learning_rate": 6.647658541141735e-05,
+ "loss": 0.779,
+ "step": 230
+ },
+ {
+ "epoch": 2.47,
+ "grad_norm": 0.35482236759964675,
+ "learning_rate": 6.566247806970119e-05,
+ "loss": 0.7936,
+ "step": 231
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 0.3318703205331905,
+ "learning_rate": 6.485094262002529e-05,
+ "loss": 0.7721,
+ "step": 232
+ },
+ {
+ "epoch": 2.49,
+ "grad_norm": 0.313412585518615,
+ "learning_rate": 6.404203984667019e-05,
+ "loss": 0.7333,
+ "step": 233
+ },
+ {
+ "epoch": 2.5,
+ "grad_norm": 0.3389693444254627,
+ "learning_rate": 6.323583033672799e-05,
+ "loss": 0.6991,
+ "step": 234
+ },
+ {
+ "epoch": 2.51,
+ "grad_norm": 0.33056782619334757,
+ "learning_rate": 6.243237447556449e-05,
+ "loss": 0.7872,
+ "step": 235
+ },
+ {
+ "epoch": 2.52,
+ "grad_norm": 0.3064085209522584,
+ "learning_rate": 6.163173244229619e-05,
+ "loss": 0.7713,
+ "step": 236
+ },
+ {
+ "epoch": 2.53,
+ "grad_norm": 0.3109445125421656,
+ "learning_rate": 6.083396420528298e-05,
+ "loss": 0.8228,
+ "step": 237
+ },
+ {
+ "epoch": 2.54,
+ "grad_norm": 0.35767207742703394,
+ "learning_rate": 6.0039129517636435e-05,
+ "loss": 0.8167,
+ "step": 238
+ },
+ {
+ "epoch": 2.55,
+ "grad_norm": 0.32869196909020376,
+ "learning_rate": 5.924728791274432e-05,
+ "loss": 0.7893,
+ "step": 239
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 0.31178216743238674,
+ "learning_rate": 5.845849869981137e-05,
+ "loss": 0.7354,
+ "step": 240
+ },
+ {
+ "epoch": 2.56,
+ "eval_loss": 1.1470853090286255,
+ "eval_runtime": 119.0749,
+ "eval_samples_per_second": 8.39,
+ "eval_steps_per_second": 0.529,
+ "step": 240
+ },
+ {
+ "epoch": 2.57,
+ "grad_norm": 0.3146586486940167,
+ "learning_rate": 5.7672820959417254e-05,
+ "loss": 0.785,
+ "step": 241
+ },
+ {
+ "epoch": 2.58,
+ "grad_norm": 0.3309473634570162,
+ "learning_rate": 5.68903135390912e-05,
+ "loss": 0.7007,
+ "step": 242
+ },
+ {
+ "epoch": 2.59,
+ "grad_norm": 0.2927704203363025,
+ "learning_rate": 5.611103504890444e-05,
+ "loss": 0.778,
+ "step": 243
+ },
+ {
+ "epoch": 2.6,
+ "grad_norm": 0.31346541530480915,
+ "learning_rate": 5.533504385708024e-05,
+ "loss": 0.7272,
+ "step": 244
+ },
+ {
+ "epoch": 2.62,
+ "grad_norm": 0.2996345434845278,
+ "learning_rate": 5.456239808562209e-05,
+ "loss": 0.8091,
+ "step": 245
+ },
+ {
+ "epoch": 2.63,
+ "grad_norm": 0.29407937930772826,
+ "learning_rate": 5.379315560596038e-05,
+ "loss": 0.7666,
+ "step": 246
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.30530254935425627,
+ "learning_rate": 5.3027374034617785e-05,
+ "loss": 0.7982,
+ "step": 247
+ },
+ {
+ "epoch": 2.65,
+ "grad_norm": 0.3298149075133802,
+ "learning_rate": 5.226511072889371e-05,
+ "loss": 0.7962,
+ "step": 248
+ },
+ {
+ "epoch": 2.66,
+ "grad_norm": 0.33155001378615223,
+ "learning_rate": 5.1506422782568345e-05,
+ "loss": 0.8087,
+ "step": 249
+ },
+ {
+ "epoch": 2.67,
+ "grad_norm": 0.32891369446509405,
+ "learning_rate": 5.0751367021626215e-05,
+ "loss": 0.7702,
+ "step": 250
+ },
+ {
+ "epoch": 2.68,
+ "grad_norm": 0.3042328939887202,
+ "learning_rate": 5.000000000000002e-05,
+ "loss": 0.7924,
+ "step": 251
+ },
+ {
+ "epoch": 2.69,
+ "grad_norm": 0.3037799376581133,
+ "learning_rate": 4.9252377995334444e-05,
+ "loss": 0.7852,
+ "step": 252
+ },
+ {
+ "epoch": 2.7,
+ "grad_norm": 0.3435430445603929,
+ "learning_rate": 4.85085570047713e-05,
+ "loss": 0.7501,
+ "step": 253
+ },
+ {
+ "epoch": 2.71,
+ "grad_norm": 0.3072160193979946,
+ "learning_rate": 4.776859274075506e-05,
+ "loss": 0.7462,
+ "step": 254
+ },
+ {
+ "epoch": 2.72,
+ "grad_norm": 0.3223586439500028,
+ "learning_rate": 4.703254062686017e-05,
+ "loss": 0.775,
+ "step": 255
+ },
+ {
+ "epoch": 2.73,
+ "grad_norm": 0.3270406403084203,
+ "learning_rate": 4.630045579363957e-05,
+ "loss": 0.8306,
+ "step": 256
+ },
+ {
+ "epoch": 2.74,
+ "grad_norm": 0.3360192842512657,
+ "learning_rate": 4.557239307449561e-05,
+ "loss": 0.7697,
+ "step": 257
+ },
+ {
+ "epoch": 2.75,
+ "grad_norm": 0.34282816479900324,
+ "learning_rate": 4.484840700157295e-05,
+ "loss": 0.7654,
+ "step": 258
+ },
+ {
+ "epoch": 2.77,
+ "grad_norm": 0.30039142762313786,
+ "learning_rate": 4.412855180167406e-05,
+ "loss": 0.7703,
+ "step": 259
+ },
+ {
+ "epoch": 2.78,
+ "grad_norm": 0.34307884673711425,
+ "learning_rate": 4.3412881392197526e-05,
+ "loss": 0.7993,
+ "step": 260
+ },
+ {
+ "epoch": 2.79,
+ "grad_norm": 0.33685538845268104,
+ "learning_rate": 4.270144937709981e-05,
+ "loss": 0.7866,
+ "step": 261
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 0.33166767859224683,
+ "learning_rate": 4.19943090428802e-05,
+ "loss": 0.8083,
+ "step": 262
+ },
+ {
+ "epoch": 2.81,
+ "grad_norm": 0.3086370003245581,
+ "learning_rate": 4.129151335458957e-05,
+ "loss": 0.7938,
+ "step": 263
+ },
+ {
+ "epoch": 2.82,
+ "grad_norm": 0.3715649674817313,
+ "learning_rate": 4.059311495186338e-05,
+ "loss": 0.7577,
+ "step": 264
+ },
+ {
+ "epoch": 2.82,
+ "eval_loss": 1.1478512287139893,
+ "eval_runtime": 119.1178,
+ "eval_samples_per_second": 8.387,
+ "eval_steps_per_second": 0.529,
+ "step": 264
+ },
+ {
+ "epoch": 2.83,
+ "grad_norm": 0.3298033298390841,
+ "learning_rate": 3.9899166144978904e-05,
+ "loss": 0.8296,
+ "step": 265
+ },
+ {
+ "epoch": 2.84,
+ "grad_norm": 0.3294808666769515,
+ "learning_rate": 3.920971891093718e-05,
+ "loss": 0.8206,
+ "step": 266
+ },
+ {
+ "epoch": 2.85,
+ "grad_norm": 0.3239672501165848,
+ "learning_rate": 3.852482488956992e-05,
+ "loss": 0.8116,
+ "step": 267
+ },
+ {
+ "epoch": 2.86,
+ "grad_norm": 0.3286742994048133,
+ "learning_rate": 3.784453537967161e-05,
+ "loss": 0.8096,
+ "step": 268
+ },
+ {
+ "epoch": 2.87,
+ "grad_norm": 0.31259050250842946,
+ "learning_rate": 3.7168901335157315e-05,
+ "loss": 0.7669,
+ "step": 269
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 0.3308991711135206,
+ "learning_rate": 3.649797336124615e-05,
+ "loss": 0.8041,
+ "step": 270
+ },
+ {
+ "epoch": 2.89,
+ "grad_norm": 0.32757727002633424,
+ "learning_rate": 3.583180171067101e-05,
+ "loss": 0.7673,
+ "step": 271
+ },
+ {
+ "epoch": 2.9,
+ "grad_norm": 0.3342551756453125,
+ "learning_rate": 3.517043627991441e-05,
+ "loss": 0.8005,
+ "step": 272
+ },
+ {
+ "epoch": 2.91,
+ "grad_norm": 0.31643754309861705,
+ "learning_rate": 3.45139266054715e-05,
+ "loss": 0.787,
+ "step": 273
+ },
+ {
+ "epoch": 2.93,
+ "grad_norm": 0.3140452683879005,
+ "learning_rate": 3.3862321860139576e-05,
+ "loss": 0.7888,
+ "step": 274
+ },
+ {
+ "epoch": 2.94,
+ "grad_norm": 0.30706221155036223,
+ "learning_rate": 3.3215670849335155e-05,
+ "loss": 0.827,
+ "step": 275
+ },
+ {
+ "epoch": 2.95,
+ "grad_norm": 0.3185483102727301,
+ "learning_rate": 3.257402200743821e-05,
+ "loss": 0.7779,
+ "step": 276
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 0.3032818796307545,
+ "learning_rate": 3.19374233941647e-05,
+ "loss": 0.7993,
+ "step": 277
+ },
+ {
+ "epoch": 2.97,
+ "grad_norm": 0.3057758504695884,
+ "learning_rate": 3.130592269096671e-05,
+ "loss": 0.768,
+ "step": 278
+ },
+ {
+ "epoch": 2.98,
+ "grad_norm": 0.3245404038219604,
+ "learning_rate": 3.0679567197461134e-05,
+ "loss": 0.7706,
+ "step": 279
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 372,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 4,
+ "save_steps": 93,
+ "total_flos": 5.168039211319689e+18,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-279/training_args.bin b/checkpoint-279/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..bc11601d02e30533659dbf12ba92faed1ba25b5a
--- /dev/null
+++ b/checkpoint-279/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c56515a18cd914d4eee44c09952d3a756ea623b0b6e69e8dfaeb0dbc7b665f46
+size 6776
diff --git a/checkpoint-279/zero_to_fp32.py b/checkpoint-279/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..49b846633d6eb1e836e34681e44033581f4edb7b
--- /dev/null
+++ b/checkpoint-279/zero_to_fp32.py
@@ -0,0 +1,592 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+ total_files = len(files)
+ state_dicts = []
+ for f in files:
+ state_dict = torch.load(f, map_location=device)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ if zero_stage <= 2:
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ elif zero_stage == 3:
+ # if there is more than one param group, there will be multiple flattened tensors - one
+ # flattened tensor per group - for simplicity merge them into a single tensor
+ #
+ # XXX: could make the script more memory efficient for when there are multiple groups - it
+ # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+ fp32_flat_groups = [
+ torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+ ]
+
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ model_files = get_model_state_files(ds_checkpoint_dir)
+
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ for name, shape in param_shapes.items():
+
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # XXX: memory usage doubles here
+ state_dict[name] = torch.cat(
+ tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+ 0).narrow(0, 0, unpartitioned_numel).view(shape)
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+ you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ """
+
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+ print(f"Saving fp32 state dict to {output_file}")
+ torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info(f"Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info(f"Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument(
+ "output_file",
+ type=str,
+ help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)
diff --git a/checkpoint-336/README.md b/checkpoint-336/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..1d1a367c446e3a5f9a585222f3bda62c8b677d2b
--- /dev/null
+++ b/checkpoint-336/README.md
@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: google/gemma-2b
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.9.0
\ No newline at end of file
diff --git a/checkpoint-336/adapter_config.json b/checkpoint-336/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a11027e1f94ce0414cb1f976775603b02ca0a82
--- /dev/null
+++ b/checkpoint-336/adapter_config.json
@@ -0,0 +1,33 @@
+{
+ "alpha_pattern": {},
+ "auto_mapping": null,
+ "base_model_name_or_path": "google/gemma-2b",
+ "bias": "none",
+ "fan_in_fan_out": null,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "v_proj",
+ "down_proj",
+ "gate_proj",
+ "k_proj",
+ "o_proj"
+ ],
+ "task_type": "CAUSAL_LM",
+ "use_dora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-336/adapter_model.safetensors b/checkpoint-336/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..88007c5ab9f91dae1bc4ebc890c4c4c690721391
--- /dev/null
+++ b/checkpoint-336/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ce5c9479f7b4e2f4f1c71ed29d0ec95f79e1731de4be9d3f7759abe3043fcdc
+size 78480320
diff --git a/checkpoint-336/global_step336/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-336/global_step336/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f616a6d85e21a06963728d0c146c064a0bb10e0
--- /dev/null
+++ b/checkpoint-336/global_step336/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f204930ec4f2a105b656f8596b32abc5228db4def6b1aa8c6f63fe8c492820e
+size 58886928
diff --git a/checkpoint-336/global_step336/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-336/global_step336/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e801efc107d09178bacb23707472c4f4cc14c46e
--- /dev/null
+++ b/checkpoint-336/global_step336/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd427c55f17c0510ec2ed53fe5e319eb0a2c4761d4083df28d11ba7aa84e5a15
+size 58885968
diff --git a/checkpoint-336/global_step336/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/checkpoint-336/global_step336/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fa0bcde701b60ab4d2f01a933057b3ea69842e19
--- /dev/null
+++ b/checkpoint-336/global_step336/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6271e5b9edc1d160ad0326ac1a89d8d44ef09363904f40271525aff81aa3b01d
+size 58886992
diff --git a/checkpoint-336/global_step336/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/checkpoint-336/global_step336/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c1bfd73d107a2b365992cc375a77a08da8ce1a82
--- /dev/null
+++ b/checkpoint-336/global_step336/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5abf7c61e69335a8e881c7220e7017eb5372fdf817a3b0d26486e4faab795701
+size 58886032
diff --git a/checkpoint-336/global_step336/mp_rank_00_model_states.pt b/checkpoint-336/global_step336/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5348de636f4e1ff9b365c04b03562adcc211d530
--- /dev/null
+++ b/checkpoint-336/global_step336/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dbda4b13cb1e71570782ac3ce184727dbacb34070d7b08deeb937890375555c
+size 1159049922
diff --git a/checkpoint-336/latest b/checkpoint-336/latest
new file mode 100644
index 0000000000000000000000000000000000000000..2318a7b72334dbaaeb3296df1453d0694267c1df
--- /dev/null
+++ b/checkpoint-336/latest
@@ -0,0 +1 @@
+global_step336
\ No newline at end of file
diff --git a/checkpoint-336/rng_state_0.pth b/checkpoint-336/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1eae54b3e473b47c36ca4eb73a37d8438e940599
--- /dev/null
+++ b/checkpoint-336/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28b9cac536dcc2f0fcb0db1a7ed44d898a5e257f0e6a2dde4782893acb56ce7d
+size 15024
diff --git a/checkpoint-336/rng_state_1.pth b/checkpoint-336/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1648873632b7b01849d076effee4e57a984deaea
--- /dev/null
+++ b/checkpoint-336/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3ee31ce56c4f2248ab7aaf5beaf8d895447d28644df750b83cc2177262498de
+size 15024
diff --git a/checkpoint-336/rng_state_2.pth b/checkpoint-336/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..58b24b2dcc40b678d90990ccde759a13b6a17d27
--- /dev/null
+++ b/checkpoint-336/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a615e7b3e06287a0e82a15b753b1c48c658347992fbb7d59ee5836d824655ebd
+size 15024
diff --git a/checkpoint-336/rng_state_3.pth b/checkpoint-336/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..33127e14222c12655f914d9b52dc3f8c4985b09d
--- /dev/null
+++ b/checkpoint-336/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe5c5388f4cf688aa51717160bed97071e825a07ba7d9a22897241c258de91d9
+size 15024
diff --git a/checkpoint-336/scheduler.pt b/checkpoint-336/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a68d06deacf6d087bdaf7905fa20eeaa6368b72f
--- /dev/null
+++ b/checkpoint-336/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:033700c231840b794630147afe6dca04265ec61bb681c241b2e3012bcb9cc8a3
+size 1064
diff --git a/checkpoint-336/trainer_state.json b/checkpoint-336/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..66c93d537aceb5a86d3b8101462f232903cd385e
--- /dev/null
+++ b/checkpoint-336/trainer_state.json
@@ -0,0 +1,2477 @@
+{
+ "best_metric": 1.203959345817566,
+ "best_model_checkpoint": "./out/checkpoint-112",
+ "epoch": 2.991097922848665,
+ "eval_steps": 28,
+ "global_step": 336,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.01,
+ "grad_norm": 4.313233023002325,
+ "learning_rate": 1.8181818181818182e-05,
+ "loss": 1.9528,
+ "step": 1
+ },
+ {
+ "epoch": 0.01,
+ "eval_loss": 2.1875686645507812,
+ "eval_runtime": 12.8608,
+ "eval_samples_per_second": 23.327,
+ "eval_steps_per_second": 2.955,
+ "step": 1
+ },
+ {
+ "epoch": 0.02,
+ "grad_norm": 4.039172290955229,
+ "learning_rate": 3.6363636363636364e-05,
+ "loss": 1.8358,
+ "step": 2
+ },
+ {
+ "epoch": 0.03,
+ "grad_norm": 4.504705512003857,
+ "learning_rate": 5.4545454545454546e-05,
+ "loss": 2.0207,
+ "step": 3
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 4.591862504847867,
+ "learning_rate": 7.272727272727273e-05,
+ "loss": 1.979,
+ "step": 4
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 3.812893581399005,
+ "learning_rate": 9.090909090909092e-05,
+ "loss": 1.8356,
+ "step": 5
+ },
+ {
+ "epoch": 0.05,
+ "grad_norm": 0.42886752872747064,
+ "learning_rate": 0.00010909090909090909,
+ "loss": 1.6722,
+ "step": 6
+ },
+ {
+ "epoch": 0.06,
+ "grad_norm": 0.22497294481851865,
+ "learning_rate": 0.00012727272727272728,
+ "loss": 1.6711,
+ "step": 7
+ },
+ {
+ "epoch": 0.07,
+ "grad_norm": 0.20955259847301927,
+ "learning_rate": 0.00014545454545454546,
+ "loss": 1.8546,
+ "step": 8
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 0.2200095325539683,
+ "learning_rate": 0.00016363636363636366,
+ "loss": 1.7538,
+ "step": 9
+ },
+ {
+ "epoch": 0.09,
+ "grad_norm": 0.19187339879899318,
+ "learning_rate": 0.00018181818181818183,
+ "loss": 1.6137,
+ "step": 10
+ },
+ {
+ "epoch": 0.1,
+ "grad_norm": 0.2113395673717837,
+ "learning_rate": 0.0002,
+ "loss": 1.5225,
+ "step": 11
+ },
+ {
+ "epoch": 0.11,
+ "grad_norm": 0.17673768408382828,
+ "learning_rate": 0.00019999741592564903,
+ "loss": 1.5303,
+ "step": 12
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 0.24120852820548402,
+ "learning_rate": 0.00019998966383614488,
+ "loss": 1.5089,
+ "step": 13
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 0.3089489160535682,
+ "learning_rate": 0.00019997674413212708,
+ "loss": 1.4525,
+ "step": 14
+ },
+ {
+ "epoch": 0.13,
+ "grad_norm": 0.2656143410731927,
+ "learning_rate": 0.00019995865748130516,
+ "loss": 1.4648,
+ "step": 15
+ },
+ {
+ "epoch": 0.14,
+ "grad_norm": 3.769410316227205,
+ "learning_rate": 0.0001999354048184241,
+ "loss": 1.3439,
+ "step": 16
+ },
+ {
+ "epoch": 0.15,
+ "grad_norm": 0.32102180658823753,
+ "learning_rate": 0.00019990698734521613,
+ "loss": 1.4644,
+ "step": 17
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 0.22094428128919438,
+ "learning_rate": 0.0001998734065303385,
+ "loss": 1.1927,
+ "step": 18
+ },
+ {
+ "epoch": 0.17,
+ "grad_norm": 0.22344487218098863,
+ "learning_rate": 0.00019983466410929764,
+ "loss": 1.2916,
+ "step": 19
+ },
+ {
+ "epoch": 0.18,
+ "grad_norm": 0.25036262498479456,
+ "learning_rate": 0.0001997907620843595,
+ "loss": 1.2982,
+ "step": 20
+ },
+ {
+ "epoch": 0.19,
+ "grad_norm": 0.22671119151539426,
+ "learning_rate": 0.00019974170272444604,
+ "loss": 1.2146,
+ "step": 21
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 0.259249080403425,
+ "learning_rate": 0.00019968748856501788,
+ "loss": 1.2072,
+ "step": 22
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 0.23538477651406017,
+ "learning_rate": 0.00019962812240794343,
+ "loss": 1.3281,
+ "step": 23
+ },
+ {
+ "epoch": 0.21,
+ "grad_norm": 0.2659115087625978,
+ "learning_rate": 0.000199563607321354,
+ "loss": 1.1396,
+ "step": 24
+ },
+ {
+ "epoch": 0.22,
+ "grad_norm": 0.23617264858854836,
+ "learning_rate": 0.0001994939466394851,
+ "loss": 1.1389,
+ "step": 25
+ },
+ {
+ "epoch": 0.23,
+ "grad_norm": 0.20514227454180176,
+ "learning_rate": 0.00019941914396250446,
+ "loss": 1.249,
+ "step": 26
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.19660894225830144,
+ "learning_rate": 0.00019933920315632557,
+ "loss": 1.1776,
+ "step": 27
+ },
+ {
+ "epoch": 0.25,
+ "grad_norm": 0.2067663909729571,
+ "learning_rate": 0.00019925412835240826,
+ "loss": 1.1327,
+ "step": 28
+ },
+ {
+ "epoch": 0.25,
+ "eval_loss": 1.2991960048675537,
+ "eval_runtime": 13.153,
+ "eval_samples_per_second": 22.808,
+ "eval_steps_per_second": 2.889,
+ "step": 28
+ },
+ {
+ "epoch": 0.26,
+ "grad_norm": 0.1816588361901526,
+ "learning_rate": 0.0001991639239475448,
+ "loss": 1.1247,
+ "step": 29
+ },
+ {
+ "epoch": 0.27,
+ "grad_norm": 0.19626955153633807,
+ "learning_rate": 0.00019906859460363307,
+ "loss": 1.1212,
+ "step": 30
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 0.21084275590405852,
+ "learning_rate": 0.00019896814524743528,
+ "loss": 0.9927,
+ "step": 31
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 0.16560054949456768,
+ "learning_rate": 0.0001988625810703235,
+ "loss": 1.1249,
+ "step": 32
+ },
+ {
+ "epoch": 0.29,
+ "grad_norm": 0.14950879528294536,
+ "learning_rate": 0.0001987519075280114,
+ "loss": 1.1401,
+ "step": 33
+ },
+ {
+ "epoch": 0.3,
+ "grad_norm": 0.1777966882651237,
+ "learning_rate": 0.00019863613034027224,
+ "loss": 1.0769,
+ "step": 34
+ },
+ {
+ "epoch": 0.31,
+ "grad_norm": 0.1480537272052743,
+ "learning_rate": 0.00019851525549064323,
+ "loss": 1.0686,
+ "step": 35
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.16911906750319078,
+ "learning_rate": 0.00019838928922611632,
+ "loss": 1.0253,
+ "step": 36
+ },
+ {
+ "epoch": 0.33,
+ "grad_norm": 0.15987682972555176,
+ "learning_rate": 0.00019825823805681543,
+ "loss": 1.0609,
+ "step": 37
+ },
+ {
+ "epoch": 0.34,
+ "grad_norm": 0.15757332939676763,
+ "learning_rate": 0.0001981221087556598,
+ "loss": 1.1086,
+ "step": 38
+ },
+ {
+ "epoch": 0.35,
+ "grad_norm": 0.13201845744757537,
+ "learning_rate": 0.00019798090835801418,
+ "loss": 1.073,
+ "step": 39
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 0.12544508015984754,
+ "learning_rate": 0.00019783464416132506,
+ "loss": 1.0633,
+ "step": 40
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 0.14645820383886451,
+ "learning_rate": 0.00019768332372474366,
+ "loss": 1.0653,
+ "step": 41
+ },
+ {
+ "epoch": 0.37,
+ "grad_norm": 0.14814101902137117,
+ "learning_rate": 0.00019752695486873517,
+ "loss": 1.0937,
+ "step": 42
+ },
+ {
+ "epoch": 0.38,
+ "grad_norm": 0.13888915595055443,
+ "learning_rate": 0.00019736554567467452,
+ "loss": 1.0462,
+ "step": 43
+ },
+ {
+ "epoch": 0.39,
+ "grad_norm": 0.13185349806639524,
+ "learning_rate": 0.00019719910448442893,
+ "loss": 1.2177,
+ "step": 44
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.15271046712350847,
+ "learning_rate": 0.00019702763989992662,
+ "loss": 1.0237,
+ "step": 45
+ },
+ {
+ "epoch": 0.41,
+ "grad_norm": 0.17053588557430902,
+ "learning_rate": 0.00019685116078271223,
+ "loss": 1.0038,
+ "step": 46
+ },
+ {
+ "epoch": 0.42,
+ "grad_norm": 0.15641087356577812,
+ "learning_rate": 0.00019666967625348906,
+ "loss": 1.0886,
+ "step": 47
+ },
+ {
+ "epoch": 0.43,
+ "grad_norm": 0.1544028594191567,
+ "learning_rate": 0.00019648319569164736,
+ "loss": 1.1378,
+ "step": 48
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 0.14794885994140625,
+ "learning_rate": 0.00019629172873477995,
+ "loss": 1.1495,
+ "step": 49
+ },
+ {
+ "epoch": 0.45,
+ "grad_norm": 0.1577684884028266,
+ "learning_rate": 0.0001960952852781838,
+ "loss": 1.0782,
+ "step": 50
+ },
+ {
+ "epoch": 0.45,
+ "grad_norm": 0.15961044045091288,
+ "learning_rate": 0.0001958938754743489,
+ "loss": 1.0107,
+ "step": 51
+ },
+ {
+ "epoch": 0.46,
+ "grad_norm": 0.14486696586022083,
+ "learning_rate": 0.0001956875097324334,
+ "loss": 1.0494,
+ "step": 52
+ },
+ {
+ "epoch": 0.47,
+ "grad_norm": 0.14250413725518896,
+ "learning_rate": 0.00019547619871772574,
+ "loss": 1.039,
+ "step": 53
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.1196720279125328,
+ "learning_rate": 0.00019525995335109334,
+ "loss": 1.0966,
+ "step": 54
+ },
+ {
+ "epoch": 0.49,
+ "grad_norm": 0.14984795891635327,
+ "learning_rate": 0.0001950387848084183,
+ "loss": 1.0874,
+ "step": 55
+ },
+ {
+ "epoch": 0.5,
+ "grad_norm": 0.14891088442480416,
+ "learning_rate": 0.00019481270452001987,
+ "loss": 1.097,
+ "step": 56
+ },
+ {
+ "epoch": 0.5,
+ "eval_loss": 1.2264304161071777,
+ "eval_runtime": 13.2279,
+ "eval_samples_per_second": 22.679,
+ "eval_steps_per_second": 2.873,
+ "step": 56
+ },
+ {
+ "epoch": 0.51,
+ "grad_norm": 0.17814266552244534,
+ "learning_rate": 0.00019458172417006347,
+ "loss": 1.1372,
+ "step": 57
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 0.16125636132578247,
+ "learning_rate": 0.00019434585569595708,
+ "loss": 1.0623,
+ "step": 58
+ },
+ {
+ "epoch": 0.53,
+ "grad_norm": 0.15203437202125702,
+ "learning_rate": 0.00019410511128773418,
+ "loss": 1.0399,
+ "step": 59
+ },
+ {
+ "epoch": 0.53,
+ "grad_norm": 0.1677461135605213,
+ "learning_rate": 0.0001938595033874238,
+ "loss": 1.0884,
+ "step": 60
+ },
+ {
+ "epoch": 0.54,
+ "grad_norm": 0.13564559875683407,
+ "learning_rate": 0.0001936090446884074,
+ "loss": 1.0176,
+ "step": 61
+ },
+ {
+ "epoch": 0.55,
+ "grad_norm": 0.1521886500642157,
+ "learning_rate": 0.00019335374813476302,
+ "loss": 1.0146,
+ "step": 62
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.1410132122625916,
+ "learning_rate": 0.00019309362692059617,
+ "loss": 1.044,
+ "step": 63
+ },
+ {
+ "epoch": 0.57,
+ "grad_norm": 0.15237848179385577,
+ "learning_rate": 0.00019282869448935798,
+ "loss": 1.0354,
+ "step": 64
+ },
+ {
+ "epoch": 0.58,
+ "grad_norm": 0.13871660988504514,
+ "learning_rate": 0.00019255896453315052,
+ "loss": 1.0189,
+ "step": 65
+ },
+ {
+ "epoch": 0.59,
+ "grad_norm": 0.14863047478901453,
+ "learning_rate": 0.000192284450992019,
+ "loss": 1.0704,
+ "step": 66
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 0.13794806124403974,
+ "learning_rate": 0.0001920051680532314,
+ "loss": 1.0996,
+ "step": 67
+ },
+ {
+ "epoch": 0.61,
+ "grad_norm": 0.13030507705779365,
+ "learning_rate": 0.00019172113015054532,
+ "loss": 1.0015,
+ "step": 68
+ },
+ {
+ "epoch": 0.61,
+ "grad_norm": 0.15092494718902358,
+ "learning_rate": 0.0001914323519634619,
+ "loss": 1.0822,
+ "step": 69
+ },
+ {
+ "epoch": 0.62,
+ "grad_norm": 0.1350212989006066,
+ "learning_rate": 0.00019113884841646736,
+ "loss": 1.0197,
+ "step": 70
+ },
+ {
+ "epoch": 0.63,
+ "grad_norm": 0.18991168066586347,
+ "learning_rate": 0.00019084063467826137,
+ "loss": 1.046,
+ "step": 71
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.14884381774710187,
+ "learning_rate": 0.00019053772616097337,
+ "loss": 1.0346,
+ "step": 72
+ },
+ {
+ "epoch": 0.65,
+ "grad_norm": 0.15579311209945296,
+ "learning_rate": 0.000190230138519366,
+ "loss": 1.0505,
+ "step": 73
+ },
+ {
+ "epoch": 0.66,
+ "grad_norm": 0.16015337150592127,
+ "learning_rate": 0.000189917887650026,
+ "loss": 1.0504,
+ "step": 74
+ },
+ {
+ "epoch": 0.67,
+ "grad_norm": 0.1443969321518926,
+ "learning_rate": 0.00018960098969054255,
+ "loss": 1.0755,
+ "step": 75
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 0.15722162227095848,
+ "learning_rate": 0.00018927946101867347,
+ "loss": 1.0541,
+ "step": 76
+ },
+ {
+ "epoch": 0.69,
+ "grad_norm": 0.17009697584926559,
+ "learning_rate": 0.0001889533182514986,
+ "loss": 1.0231,
+ "step": 77
+ },
+ {
+ "epoch": 0.69,
+ "grad_norm": 0.1256822726781221,
+ "learning_rate": 0.0001886225782445612,
+ "loss": 0.8814,
+ "step": 78
+ },
+ {
+ "epoch": 0.7,
+ "grad_norm": 0.14019958069756655,
+ "learning_rate": 0.00018828725809099655,
+ "loss": 1.0277,
+ "step": 79
+ },
+ {
+ "epoch": 0.71,
+ "grad_norm": 0.17159459150063183,
+ "learning_rate": 0.0001879473751206489,
+ "loss": 0.9495,
+ "step": 80
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.146430011834186,
+ "learning_rate": 0.00018760294689917553,
+ "loss": 1.0598,
+ "step": 81
+ },
+ {
+ "epoch": 0.73,
+ "grad_norm": 0.16834256802992476,
+ "learning_rate": 0.00018725399122713912,
+ "loss": 1.0237,
+ "step": 82
+ },
+ {
+ "epoch": 0.74,
+ "grad_norm": 0.15663699267164208,
+ "learning_rate": 0.00018690052613908772,
+ "loss": 0.939,
+ "step": 83
+ },
+ {
+ "epoch": 0.75,
+ "grad_norm": 0.15655985150409854,
+ "learning_rate": 0.0001865425699026226,
+ "loss": 1.0302,
+ "step": 84
+ },
+ {
+ "epoch": 0.75,
+ "eval_loss": 1.2143030166625977,
+ "eval_runtime": 13.2387,
+ "eval_samples_per_second": 22.661,
+ "eval_steps_per_second": 2.87,
+ "step": 84
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 0.15273470110260864,
+ "learning_rate": 0.00018618014101745442,
+ "loss": 1.0127,
+ "step": 85
+ },
+ {
+ "epoch": 0.77,
+ "grad_norm": 0.1723243680259614,
+ "learning_rate": 0.0001858132582144469,
+ "loss": 0.9306,
+ "step": 86
+ },
+ {
+ "epoch": 0.77,
+ "grad_norm": 0.14747098547446996,
+ "learning_rate": 0.00018544194045464886,
+ "loss": 1.0073,
+ "step": 87
+ },
+ {
+ "epoch": 0.78,
+ "grad_norm": 0.17208333285514918,
+ "learning_rate": 0.00018506620692831428,
+ "loss": 1.0328,
+ "step": 88
+ },
+ {
+ "epoch": 0.79,
+ "grad_norm": 0.14918051024971962,
+ "learning_rate": 0.0001846860770539105,
+ "loss": 1.0022,
+ "step": 89
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.156315164090714,
+ "learning_rate": 0.00018430157047711474,
+ "loss": 1.0293,
+ "step": 90
+ },
+ {
+ "epoch": 0.81,
+ "grad_norm": 0.2013424548288477,
+ "learning_rate": 0.00018391270706979862,
+ "loss": 0.9395,
+ "step": 91
+ },
+ {
+ "epoch": 0.82,
+ "grad_norm": 0.17909726353002614,
+ "learning_rate": 0.00018351950692900126,
+ "loss": 0.9756,
+ "step": 92
+ },
+ {
+ "epoch": 0.83,
+ "grad_norm": 0.16939245158726288,
+ "learning_rate": 0.00018312199037589068,
+ "loss": 0.9576,
+ "step": 93
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 0.14685720680893694,
+ "learning_rate": 0.00018272017795471345,
+ "loss": 1.0045,
+ "step": 94
+ },
+ {
+ "epoch": 0.85,
+ "grad_norm": 0.17464839085505987,
+ "learning_rate": 0.000182314090431733,
+ "loss": 0.9862,
+ "step": 95
+ },
+ {
+ "epoch": 0.85,
+ "grad_norm": 0.16060904136932572,
+ "learning_rate": 0.00018190374879415632,
+ "loss": 1.0022,
+ "step": 96
+ },
+ {
+ "epoch": 0.86,
+ "grad_norm": 0.18715193350083867,
+ "learning_rate": 0.00018148917424904953,
+ "loss": 1.042,
+ "step": 97
+ },
+ {
+ "epoch": 0.87,
+ "grad_norm": 0.1675573400576595,
+ "learning_rate": 0.0001810703882222415,
+ "loss": 1.0047,
+ "step": 98
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.1871466286989249,
+ "learning_rate": 0.00018064741235721687,
+ "loss": 0.9834,
+ "step": 99
+ },
+ {
+ "epoch": 0.89,
+ "grad_norm": 0.17453934867565302,
+ "learning_rate": 0.00018022026851399737,
+ "loss": 0.9649,
+ "step": 100
+ },
+ {
+ "epoch": 0.9,
+ "grad_norm": 0.15960631507184767,
+ "learning_rate": 0.0001797889787680119,
+ "loss": 0.9673,
+ "step": 101
+ },
+ {
+ "epoch": 0.91,
+ "grad_norm": 0.17844936635366368,
+ "learning_rate": 0.00017935356540895597,
+ "loss": 1.0951,
+ "step": 102
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 0.16733018789000254,
+ "learning_rate": 0.00017891405093963938,
+ "loss": 0.9954,
+ "step": 103
+ },
+ {
+ "epoch": 0.93,
+ "grad_norm": 0.17305556075296993,
+ "learning_rate": 0.00017847045807482345,
+ "loss": 0.892,
+ "step": 104
+ },
+ {
+ "epoch": 0.93,
+ "grad_norm": 0.17197614099805034,
+ "learning_rate": 0.00017802280974004716,
+ "loss": 1.0494,
+ "step": 105
+ },
+ {
+ "epoch": 0.94,
+ "grad_norm": 0.18063836817127235,
+ "learning_rate": 0.000177571129070442,
+ "loss": 1.0264,
+ "step": 106
+ },
+ {
+ "epoch": 0.95,
+ "grad_norm": 0.14597707005699143,
+ "learning_rate": 0.00017711543940953668,
+ "loss": 0.9532,
+ "step": 107
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 0.1422048149465345,
+ "learning_rate": 0.00017665576430805053,
+ "loss": 0.97,
+ "step": 108
+ },
+ {
+ "epoch": 0.97,
+ "grad_norm": 0.18313914688655572,
+ "learning_rate": 0.0001761921275226763,
+ "loss": 0.9282,
+ "step": 109
+ },
+ {
+ "epoch": 0.98,
+ "grad_norm": 0.200679751171441,
+ "learning_rate": 0.00017572455301485249,
+ "loss": 1.0,
+ "step": 110
+ },
+ {
+ "epoch": 0.99,
+ "grad_norm": 0.17700985594898055,
+ "learning_rate": 0.00017525306494952498,
+ "loss": 1.0165,
+ "step": 111
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 0.19925777202726191,
+ "learning_rate": 0.0001747776876938981,
+ "loss": 1.0346,
+ "step": 112
+ },
+ {
+ "epoch": 1.0,
+ "eval_loss": 1.203959345817566,
+ "eval_runtime": 13.2547,
+ "eval_samples_per_second": 22.634,
+ "eval_steps_per_second": 2.867,
+ "step": 112
+ },
+ {
+ "epoch": 1.01,
+ "grad_norm": 0.1606469603473709,
+ "learning_rate": 0.00017429844581617532,
+ "loss": 0.9832,
+ "step": 113
+ },
+ {
+ "epoch": 1.01,
+ "grad_norm": 0.16403912763780054,
+ "learning_rate": 0.00017381536408428948,
+ "loss": 0.9346,
+ "step": 114
+ },
+ {
+ "epoch": 1.02,
+ "grad_norm": 0.1936046893744468,
+ "learning_rate": 0.00017332846746462288,
+ "loss": 0.9382,
+ "step": 115
+ },
+ {
+ "epoch": 1.03,
+ "grad_norm": 0.14250769247239573,
+ "learning_rate": 0.0001728377811207168,
+ "loss": 0.8914,
+ "step": 116
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.17889563599797687,
+ "learning_rate": 0.00017234333041197126,
+ "loss": 0.9736,
+ "step": 117
+ },
+ {
+ "epoch": 1.05,
+ "grad_norm": 0.20288960866045594,
+ "learning_rate": 0.00017184514089233405,
+ "loss": 0.8477,
+ "step": 118
+ },
+ {
+ "epoch": 1.06,
+ "grad_norm": 0.20926349930533472,
+ "learning_rate": 0.00017134323830898037,
+ "loss": 0.9933,
+ "step": 119
+ },
+ {
+ "epoch": 1.07,
+ "grad_norm": 0.21316934416499642,
+ "learning_rate": 0.00017083764860098205,
+ "loss": 0.9168,
+ "step": 120
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 0.21654320387312692,
+ "learning_rate": 0.0001703283978979671,
+ "loss": 0.9584,
+ "step": 121
+ },
+ {
+ "epoch": 1.09,
+ "grad_norm": 0.23789742308175463,
+ "learning_rate": 0.00016981551251876904,
+ "loss": 1.0298,
+ "step": 122
+ },
+ {
+ "epoch": 1.09,
+ "grad_norm": 0.16433271793469648,
+ "learning_rate": 0.00016929901897006698,
+ "loss": 0.8833,
+ "step": 123
+ },
+ {
+ "epoch": 1.1,
+ "grad_norm": 0.16908727866207868,
+ "learning_rate": 0.0001687789439450156,
+ "loss": 1.0675,
+ "step": 124
+ },
+ {
+ "epoch": 1.11,
+ "grad_norm": 0.1670067931363302,
+ "learning_rate": 0.00016825531432186543,
+ "loss": 0.9515,
+ "step": 125
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.17777465531550865,
+ "learning_rate": 0.00016772815716257412,
+ "loss": 0.8929,
+ "step": 126
+ },
+ {
+ "epoch": 1.13,
+ "grad_norm": 0.18442783204919333,
+ "learning_rate": 0.00016719749971140754,
+ "loss": 0.8388,
+ "step": 127
+ },
+ {
+ "epoch": 1.14,
+ "grad_norm": 0.19073362304284272,
+ "learning_rate": 0.0001666633693935319,
+ "loss": 0.9584,
+ "step": 128
+ },
+ {
+ "epoch": 1.15,
+ "grad_norm": 0.20189563405135308,
+ "learning_rate": 0.00016612579381359622,
+ "loss": 1.0264,
+ "step": 129
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 0.1694138210313381,
+ "learning_rate": 0.00016558480075430594,
+ "loss": 0.9592,
+ "step": 130
+ },
+ {
+ "epoch": 1.17,
+ "grad_norm": 0.19195382946787184,
+ "learning_rate": 0.00016504041817498678,
+ "loss": 0.974,
+ "step": 131
+ },
+ {
+ "epoch": 1.18,
+ "grad_norm": 0.20684215619155688,
+ "learning_rate": 0.00016449267421013994,
+ "loss": 0.8499,
+ "step": 132
+ },
+ {
+ "epoch": 1.18,
+ "grad_norm": 0.22003490429847744,
+ "learning_rate": 0.00016394159716798807,
+ "loss": 0.9659,
+ "step": 133
+ },
+ {
+ "epoch": 1.19,
+ "grad_norm": 0.21977918206745437,
+ "learning_rate": 0.00016338721552901212,
+ "loss": 0.9213,
+ "step": 134
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.2076993903333204,
+ "learning_rate": 0.0001628295579444796,
+ "loss": 0.8119,
+ "step": 135
+ },
+ {
+ "epoch": 1.21,
+ "grad_norm": 0.2001771499954729,
+ "learning_rate": 0.0001622686532349637,
+ "loss": 0.9183,
+ "step": 136
+ },
+ {
+ "epoch": 1.22,
+ "grad_norm": 0.18671550149366203,
+ "learning_rate": 0.00016170453038885394,
+ "loss": 0.8836,
+ "step": 137
+ },
+ {
+ "epoch": 1.23,
+ "grad_norm": 0.20867427207572573,
+ "learning_rate": 0.0001611372185608578,
+ "loss": 0.9964,
+ "step": 138
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 0.20035138443113176,
+ "learning_rate": 0.0001605667470704942,
+ "loss": 0.9209,
+ "step": 139
+ },
+ {
+ "epoch": 1.25,
+ "grad_norm": 0.22696612020505577,
+ "learning_rate": 0.0001599931454005781,
+ "loss": 1.0162,
+ "step": 140
+ },
+ {
+ "epoch": 1.25,
+ "eval_loss": 1.2188584804534912,
+ "eval_runtime": 13.249,
+ "eval_samples_per_second": 22.643,
+ "eval_steps_per_second": 2.868,
+ "step": 140
+ },
+ {
+ "epoch": 1.26,
+ "grad_norm": 0.21554353495018647,
+ "learning_rate": 0.00015941644319569665,
+ "loss": 1.0487,
+ "step": 141
+ },
+ {
+ "epoch": 1.26,
+ "grad_norm": 0.22894492131909072,
+ "learning_rate": 0.00015883667026067745,
+ "loss": 0.9352,
+ "step": 142
+ },
+ {
+ "epoch": 1.27,
+ "grad_norm": 0.19145184577172686,
+ "learning_rate": 0.00015825385655904788,
+ "loss": 0.8878,
+ "step": 143
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 0.22544664152936575,
+ "learning_rate": 0.00015766803221148673,
+ "loss": 1.0,
+ "step": 144
+ },
+ {
+ "epoch": 1.29,
+ "grad_norm": 0.26000661355557114,
+ "learning_rate": 0.00015707922749426737,
+ "loss": 0.9339,
+ "step": 145
+ },
+ {
+ "epoch": 1.3,
+ "grad_norm": 0.24433845134512236,
+ "learning_rate": 0.00015648747283769317,
+ "loss": 0.9474,
+ "step": 146
+ },
+ {
+ "epoch": 1.31,
+ "grad_norm": 0.21973931169609887,
+ "learning_rate": 0.00015589279882452476,
+ "loss": 0.9357,
+ "step": 147
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 0.23929008733305812,
+ "learning_rate": 0.0001552952361883994,
+ "loss": 0.9985,
+ "step": 148
+ },
+ {
+ "epoch": 1.33,
+ "grad_norm": 0.23431856747573573,
+ "learning_rate": 0.00015469481581224272,
+ "loss": 0.8913,
+ "step": 149
+ },
+ {
+ "epoch": 1.34,
+ "grad_norm": 0.2233543327912565,
+ "learning_rate": 0.00015409156872667258,
+ "loss": 0.9877,
+ "step": 150
+ },
+ {
+ "epoch": 1.34,
+ "grad_norm": 0.21281207674183256,
+ "learning_rate": 0.0001534855261083954,
+ "loss": 0.9071,
+ "step": 151
+ },
+ {
+ "epoch": 1.35,
+ "grad_norm": 0.20314832700152685,
+ "learning_rate": 0.00015287671927859494,
+ "loss": 0.9373,
+ "step": 152
+ },
+ {
+ "epoch": 1.36,
+ "grad_norm": 0.19648565819019825,
+ "learning_rate": 0.00015226517970131343,
+ "loss": 0.9469,
+ "step": 153
+ },
+ {
+ "epoch": 1.37,
+ "grad_norm": 0.2262428264639853,
+ "learning_rate": 0.00015165093898182562,
+ "loss": 1.0066,
+ "step": 154
+ },
+ {
+ "epoch": 1.38,
+ "grad_norm": 0.22253433035020442,
+ "learning_rate": 0.00015103402886500525,
+ "loss": 0.8875,
+ "step": 155
+ },
+ {
+ "epoch": 1.39,
+ "grad_norm": 0.181161648904613,
+ "learning_rate": 0.00015041448123368455,
+ "loss": 0.9004,
+ "step": 156
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 0.20968483802367816,
+ "learning_rate": 0.00014979232810700637,
+ "loss": 0.9133,
+ "step": 157
+ },
+ {
+ "epoch": 1.41,
+ "grad_norm": 0.20540509271288435,
+ "learning_rate": 0.0001491676016387694,
+ "loss": 0.8876,
+ "step": 158
+ },
+ {
+ "epoch": 1.42,
+ "grad_norm": 0.18762795731312454,
+ "learning_rate": 0.00014854033411576659,
+ "loss": 0.933,
+ "step": 159
+ },
+ {
+ "epoch": 1.42,
+ "grad_norm": 0.23223345997338857,
+ "learning_rate": 0.00014791055795611624,
+ "loss": 0.9182,
+ "step": 160
+ },
+ {
+ "epoch": 1.43,
+ "grad_norm": 0.21932384461027146,
+ "learning_rate": 0.00014727830570758678,
+ "loss": 0.9514,
+ "step": 161
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.21819663730951108,
+ "learning_rate": 0.0001466436100459146,
+ "loss": 0.9162,
+ "step": 162
+ },
+ {
+ "epoch": 1.45,
+ "grad_norm": 0.2325813323476676,
+ "learning_rate": 0.00014600650377311522,
+ "loss": 0.9308,
+ "step": 163
+ },
+ {
+ "epoch": 1.46,
+ "grad_norm": 0.2568337182939043,
+ "learning_rate": 0.0001453670198157883,
+ "loss": 0.9995,
+ "step": 164
+ },
+ {
+ "epoch": 1.47,
+ "grad_norm": 0.22578454460723413,
+ "learning_rate": 0.00014472519122341566,
+ "loss": 0.9052,
+ "step": 165
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 0.23564258958796755,
+ "learning_rate": 0.00014408105116665336,
+ "loss": 0.9714,
+ "step": 166
+ },
+ {
+ "epoch": 1.49,
+ "grad_norm": 0.24266133562839415,
+ "learning_rate": 0.00014343463293561734,
+ "loss": 0.9219,
+ "step": 167
+ },
+ {
+ "epoch": 1.5,
+ "grad_norm": 0.23472454708184465,
+ "learning_rate": 0.00014278596993816308,
+ "loss": 0.8762,
+ "step": 168
+ },
+ {
+ "epoch": 1.5,
+ "eval_loss": 1.2197421789169312,
+ "eval_runtime": 13.2616,
+ "eval_samples_per_second": 22.622,
+ "eval_steps_per_second": 2.865,
+ "step": 168
+ },
+ {
+ "epoch": 1.5,
+ "grad_norm": 0.23623633375452713,
+ "learning_rate": 0.00014213509569815884,
+ "loss": 0.8809,
+ "step": 169
+ },
+ {
+ "epoch": 1.51,
+ "grad_norm": 0.25344275204523486,
+ "learning_rate": 0.00014148204385375321,
+ "loss": 0.7972,
+ "step": 170
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 0.23111396119549557,
+ "learning_rate": 0.0001408268481556366,
+ "loss": 0.8228,
+ "step": 171
+ },
+ {
+ "epoch": 1.53,
+ "grad_norm": 0.2510618369255398,
+ "learning_rate": 0.00014016954246529696,
+ "loss": 0.8849,
+ "step": 172
+ },
+ {
+ "epoch": 1.54,
+ "grad_norm": 0.2764366116622668,
+ "learning_rate": 0.0001395101607532698,
+ "loss": 0.8936,
+ "step": 173
+ },
+ {
+ "epoch": 1.55,
+ "grad_norm": 0.24325811719582827,
+ "learning_rate": 0.00013884873709738257,
+ "loss": 0.8602,
+ "step": 174
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 0.213781513838486,
+ "learning_rate": 0.00013818530568099327,
+ "loss": 0.9492,
+ "step": 175
+ },
+ {
+ "epoch": 1.57,
+ "grad_norm": 0.2397396374239057,
+ "learning_rate": 0.00013751990079122412,
+ "loss": 1.0499,
+ "step": 176
+ },
+ {
+ "epoch": 1.58,
+ "grad_norm": 0.21579907170368723,
+ "learning_rate": 0.00013685255681718922,
+ "loss": 0.9438,
+ "step": 177
+ },
+ {
+ "epoch": 1.58,
+ "grad_norm": 0.2359312681928786,
+ "learning_rate": 0.0001361833082482175,
+ "loss": 0.9289,
+ "step": 178
+ },
+ {
+ "epoch": 1.59,
+ "grad_norm": 0.2618189093396496,
+ "learning_rate": 0.0001355121896720703,
+ "loss": 0.981,
+ "step": 179
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.20876513773174135,
+ "learning_rate": 0.00013483923577315348,
+ "loss": 0.82,
+ "step": 180
+ },
+ {
+ "epoch": 1.61,
+ "grad_norm": 0.22162748553995645,
+ "learning_rate": 0.00013416448133072526,
+ "loss": 1.0131,
+ "step": 181
+ },
+ {
+ "epoch": 1.62,
+ "grad_norm": 0.20975549982451164,
+ "learning_rate": 0.00013348796121709862,
+ "loss": 0.8763,
+ "step": 182
+ },
+ {
+ "epoch": 1.63,
+ "grad_norm": 0.22840397707525473,
+ "learning_rate": 0.00013280971039583906,
+ "loss": 0.949,
+ "step": 183
+ },
+ {
+ "epoch": 1.64,
+ "grad_norm": 0.23384636230161737,
+ "learning_rate": 0.0001321297639199575,
+ "loss": 0.9567,
+ "step": 184
+ },
+ {
+ "epoch": 1.65,
+ "grad_norm": 0.22905979409902957,
+ "learning_rate": 0.000131448156930099,
+ "loss": 0.9153,
+ "step": 185
+ },
+ {
+ "epoch": 1.66,
+ "grad_norm": 0.27620894683694563,
+ "learning_rate": 0.0001307649246527263,
+ "loss": 0.8246,
+ "step": 186
+ },
+ {
+ "epoch": 1.66,
+ "grad_norm": 0.23004170633106227,
+ "learning_rate": 0.0001300801023982995,
+ "loss": 1.0181,
+ "step": 187
+ },
+ {
+ "epoch": 1.67,
+ "grad_norm": 0.2219849136264378,
+ "learning_rate": 0.00012939372555945112,
+ "loss": 0.9535,
+ "step": 188
+ },
+ {
+ "epoch": 1.68,
+ "grad_norm": 0.24458750452490116,
+ "learning_rate": 0.0001287058296091567,
+ "loss": 0.8968,
+ "step": 189
+ },
+ {
+ "epoch": 1.69,
+ "grad_norm": 0.2564337740159555,
+ "learning_rate": 0.00012801645009890195,
+ "loss": 0.7955,
+ "step": 190
+ },
+ {
+ "epoch": 1.7,
+ "grad_norm": 0.24100850371438767,
+ "learning_rate": 0.0001273256226568451,
+ "loss": 0.9235,
+ "step": 191
+ },
+ {
+ "epoch": 1.71,
+ "grad_norm": 0.24757089527873732,
+ "learning_rate": 0.00012663338298597563,
+ "loss": 1.007,
+ "step": 192
+ },
+ {
+ "epoch": 1.72,
+ "grad_norm": 0.24701038583742888,
+ "learning_rate": 0.00012593976686226904,
+ "loss": 0.9885,
+ "step": 193
+ },
+ {
+ "epoch": 1.73,
+ "grad_norm": 0.26373721125634964,
+ "learning_rate": 0.0001252448101328381,
+ "loss": 0.8785,
+ "step": 194
+ },
+ {
+ "epoch": 1.74,
+ "grad_norm": 0.2227761464470136,
+ "learning_rate": 0.00012454854871407994,
+ "loss": 0.8806,
+ "step": 195
+ },
+ {
+ "epoch": 1.74,
+ "grad_norm": 0.2283950634350429,
+ "learning_rate": 0.00012385101858982005,
+ "loss": 0.9053,
+ "step": 196
+ },
+ {
+ "epoch": 1.74,
+ "eval_loss": 1.2198154926300049,
+ "eval_runtime": 13.2208,
+ "eval_samples_per_second": 22.692,
+ "eval_steps_per_second": 2.874,
+ "step": 196
+ },
+ {
+ "epoch": 1.75,
+ "grad_norm": 0.23406423788354982,
+ "learning_rate": 0.00012315225580945252,
+ "loss": 0.9397,
+ "step": 197
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 0.23807045727443327,
+ "learning_rate": 0.0001224522964860769,
+ "loss": 0.9712,
+ "step": 198
+ },
+ {
+ "epoch": 1.77,
+ "grad_norm": 0.2463614808838948,
+ "learning_rate": 0.00012175117679463187,
+ "loss": 0.8558,
+ "step": 199
+ },
+ {
+ "epoch": 1.78,
+ "grad_norm": 0.24737417059302014,
+ "learning_rate": 0.00012104893297002567,
+ "loss": 0.9723,
+ "step": 200
+ },
+ {
+ "epoch": 1.79,
+ "grad_norm": 0.243750688050595,
+ "learning_rate": 0.0001203456013052634,
+ "loss": 0.964,
+ "step": 201
+ },
+ {
+ "epoch": 1.8,
+ "grad_norm": 0.24572059557106538,
+ "learning_rate": 0.00011964121814957137,
+ "loss": 0.9109,
+ "step": 202
+ },
+ {
+ "epoch": 1.81,
+ "grad_norm": 0.24044117903962453,
+ "learning_rate": 0.00011893581990651848,
+ "loss": 1.0019,
+ "step": 203
+ },
+ {
+ "epoch": 1.82,
+ "grad_norm": 0.2737568489071465,
+ "learning_rate": 0.00011822944303213486,
+ "loss": 0.8893,
+ "step": 204
+ },
+ {
+ "epoch": 1.82,
+ "grad_norm": 0.24122455882790084,
+ "learning_rate": 0.00011752212403302784,
+ "loss": 0.9162,
+ "step": 205
+ },
+ {
+ "epoch": 1.83,
+ "grad_norm": 0.28991871401626856,
+ "learning_rate": 0.00011681389946449504,
+ "loss": 0.8555,
+ "step": 206
+ },
+ {
+ "epoch": 1.84,
+ "grad_norm": 0.23767408810646548,
+ "learning_rate": 0.00011610480592863531,
+ "loss": 0.9936,
+ "step": 207
+ },
+ {
+ "epoch": 1.85,
+ "grad_norm": 0.22614733706173062,
+ "learning_rate": 0.00011539488007245702,
+ "loss": 0.916,
+ "step": 208
+ },
+ {
+ "epoch": 1.86,
+ "grad_norm": 0.22471992425846515,
+ "learning_rate": 0.00011468415858598411,
+ "loss": 0.8872,
+ "step": 209
+ },
+ {
+ "epoch": 1.87,
+ "grad_norm": 0.22675717145909688,
+ "learning_rate": 0.00011397267820035986,
+ "loss": 0.8393,
+ "step": 210
+ },
+ {
+ "epoch": 1.88,
+ "grad_norm": 0.2727459336483823,
+ "learning_rate": 0.00011326047568594851,
+ "loss": 0.8265,
+ "step": 211
+ },
+ {
+ "epoch": 1.89,
+ "grad_norm": 0.25216778031670767,
+ "learning_rate": 0.00011254758785043515,
+ "loss": 0.9939,
+ "step": 212
+ },
+ {
+ "epoch": 1.9,
+ "grad_norm": 0.269147378424304,
+ "learning_rate": 0.0001118340515369232,
+ "loss": 0.9102,
+ "step": 213
+ },
+ {
+ "epoch": 1.91,
+ "grad_norm": 0.2216178370833471,
+ "learning_rate": 0.00011111990362203033,
+ "loss": 0.8778,
+ "step": 214
+ },
+ {
+ "epoch": 1.91,
+ "grad_norm": 0.2602474934716497,
+ "learning_rate": 0.00011040518101398276,
+ "loss": 0.9454,
+ "step": 215
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 0.2658635078442998,
+ "learning_rate": 0.00010968992065070769,
+ "loss": 0.8098,
+ "step": 216
+ },
+ {
+ "epoch": 1.93,
+ "grad_norm": 0.20997905209488962,
+ "learning_rate": 0.00010897415949792427,
+ "loss": 0.9318,
+ "step": 217
+ },
+ {
+ "epoch": 1.94,
+ "grad_norm": 0.24752453752221557,
+ "learning_rate": 0.00010825793454723325,
+ "loss": 0.949,
+ "step": 218
+ },
+ {
+ "epoch": 1.95,
+ "grad_norm": 0.255579569750529,
+ "learning_rate": 0.0001075412828142051,
+ "loss": 0.915,
+ "step": 219
+ },
+ {
+ "epoch": 1.96,
+ "grad_norm": 0.23186981930561867,
+ "learning_rate": 0.0001068242413364671,
+ "loss": 0.9132,
+ "step": 220
+ },
+ {
+ "epoch": 1.97,
+ "grad_norm": 0.35685140391438824,
+ "learning_rate": 0.00010610684717178905,
+ "loss": 0.9398,
+ "step": 221
+ },
+ {
+ "epoch": 1.98,
+ "grad_norm": 0.27320389987223703,
+ "learning_rate": 0.00010538913739616816,
+ "loss": 0.857,
+ "step": 222
+ },
+ {
+ "epoch": 1.99,
+ "grad_norm": 0.2324276771141761,
+ "learning_rate": 0.00010467114910191289,
+ "loss": 0.8546,
+ "step": 223
+ },
+ {
+ "epoch": 1.99,
+ "grad_norm": 0.22820341349854167,
+ "learning_rate": 0.00010395291939572593,
+ "loss": 0.9301,
+ "step": 224
+ },
+ {
+ "epoch": 1.99,
+ "eval_loss": 1.2246263027191162,
+ "eval_runtime": 13.1981,
+ "eval_samples_per_second": 22.731,
+ "eval_steps_per_second": 2.879,
+ "step": 224
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 0.2289800489154315,
+ "learning_rate": 0.00010323448539678653,
+ "loss": 0.9922,
+ "step": 225
+ },
+ {
+ "epoch": 2.01,
+ "grad_norm": 0.2673353778680862,
+ "learning_rate": 0.00010251588423483205,
+ "loss": 0.7779,
+ "step": 226
+ },
+ {
+ "epoch": 2.02,
+ "grad_norm": 0.2420933678952559,
+ "learning_rate": 0.0001017971530482392,
+ "loss": 0.8044,
+ "step": 227
+ },
+ {
+ "epoch": 2.03,
+ "grad_norm": 0.21799264660625498,
+ "learning_rate": 0.00010107832898210439,
+ "loss": 0.8773,
+ "step": 228
+ },
+ {
+ "epoch": 2.04,
+ "grad_norm": 0.21443255695871016,
+ "learning_rate": 0.00010035944918632429,
+ "loss": 0.9031,
+ "step": 229
+ },
+ {
+ "epoch": 2.05,
+ "grad_norm": 0.23983734165788242,
+ "learning_rate": 9.96405508136757e-05,
+ "loss": 0.9014,
+ "step": 230
+ },
+ {
+ "epoch": 2.06,
+ "grad_norm": 0.27915481475799336,
+ "learning_rate": 9.892167101789564e-05,
+ "loss": 0.8853,
+ "step": 231
+ },
+ {
+ "epoch": 2.07,
+ "grad_norm": 0.2688949371564916,
+ "learning_rate": 9.820284695176082e-05,
+ "loss": 0.8452,
+ "step": 232
+ },
+ {
+ "epoch": 2.07,
+ "grad_norm": 0.2623278518867105,
+ "learning_rate": 9.748411576516794e-05,
+ "loss": 0.8612,
+ "step": 233
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 0.2710502639103885,
+ "learning_rate": 9.676551460321349e-05,
+ "loss": 0.8108,
+ "step": 234
+ },
+ {
+ "epoch": 2.09,
+ "grad_norm": 0.282572880285737,
+ "learning_rate": 9.60470806042741e-05,
+ "loss": 0.7866,
+ "step": 235
+ },
+ {
+ "epoch": 2.1,
+ "grad_norm": 0.2829396962922612,
+ "learning_rate": 9.532885089808713e-05,
+ "loss": 0.8557,
+ "step": 236
+ },
+ {
+ "epoch": 2.11,
+ "grad_norm": 0.2721172338857335,
+ "learning_rate": 9.461086260383187e-05,
+ "loss": 0.7933,
+ "step": 237
+ },
+ {
+ "epoch": 2.12,
+ "grad_norm": 0.29736638811364446,
+ "learning_rate": 9.389315282821097e-05,
+ "loss": 0.7674,
+ "step": 238
+ },
+ {
+ "epoch": 2.13,
+ "grad_norm": 0.28571679920981263,
+ "learning_rate": 9.317575866353292e-05,
+ "loss": 0.7442,
+ "step": 239
+ },
+ {
+ "epoch": 2.14,
+ "grad_norm": 0.264545167150173,
+ "learning_rate": 9.245871718579491e-05,
+ "loss": 0.8505,
+ "step": 240
+ },
+ {
+ "epoch": 2.15,
+ "grad_norm": 0.30691085134027757,
+ "learning_rate": 9.174206545276677e-05,
+ "loss": 0.7898,
+ "step": 241
+ },
+ {
+ "epoch": 2.15,
+ "grad_norm": 0.31375028121981235,
+ "learning_rate": 9.102584050207578e-05,
+ "loss": 0.7661,
+ "step": 242
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 0.28421530221837016,
+ "learning_rate": 9.031007934929236e-05,
+ "loss": 0.8328,
+ "step": 243
+ },
+ {
+ "epoch": 2.17,
+ "grad_norm": 0.25601367811173414,
+ "learning_rate": 8.959481898601728e-05,
+ "loss": 0.8281,
+ "step": 244
+ },
+ {
+ "epoch": 2.18,
+ "grad_norm": 0.2983724947729522,
+ "learning_rate": 8.888009637796968e-05,
+ "loss": 0.8567,
+ "step": 245
+ },
+ {
+ "epoch": 2.19,
+ "grad_norm": 0.2545616786933236,
+ "learning_rate": 8.81659484630768e-05,
+ "loss": 0.9151,
+ "step": 246
+ },
+ {
+ "epoch": 2.2,
+ "grad_norm": 0.23873712362647942,
+ "learning_rate": 8.745241214956483e-05,
+ "loss": 0.8818,
+ "step": 247
+ },
+ {
+ "epoch": 2.21,
+ "grad_norm": 0.285331972404065,
+ "learning_rate": 8.673952431405148e-05,
+ "loss": 0.7983,
+ "step": 248
+ },
+ {
+ "epoch": 2.22,
+ "grad_norm": 0.23897707291689843,
+ "learning_rate": 8.602732179964017e-05,
+ "loss": 0.8758,
+ "step": 249
+ },
+ {
+ "epoch": 2.23,
+ "grad_norm": 0.2830966091447457,
+ "learning_rate": 8.531584141401591e-05,
+ "loss": 0.8714,
+ "step": 250
+ },
+ {
+ "epoch": 2.23,
+ "grad_norm": 0.28872599217076506,
+ "learning_rate": 8.4605119927543e-05,
+ "loss": 0.8387,
+ "step": 251
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 0.2652236346400331,
+ "learning_rate": 8.38951940713647e-05,
+ "loss": 0.8232,
+ "step": 252
+ },
+ {
+ "epoch": 2.24,
+ "eval_loss": 1.2432794570922852,
+ "eval_runtime": 13.2405,
+ "eval_samples_per_second": 22.658,
+ "eval_steps_per_second": 2.87,
+ "step": 252
+ },
+ {
+ "epoch": 2.25,
+ "grad_norm": 0.299978013524394,
+ "learning_rate": 8.318610053550497e-05,
+ "loss": 0.7321,
+ "step": 253
+ },
+ {
+ "epoch": 2.26,
+ "grad_norm": 0.2740002835117391,
+ "learning_rate": 8.247787596697218e-05,
+ "loss": 0.7605,
+ "step": 254
+ },
+ {
+ "epoch": 2.27,
+ "grad_norm": 0.2848366030132808,
+ "learning_rate": 8.177055696786516e-05,
+ "loss": 0.8485,
+ "step": 255
+ },
+ {
+ "epoch": 2.28,
+ "grad_norm": 0.24847418856075218,
+ "learning_rate": 8.106418009348157e-05,
+ "loss": 0.7557,
+ "step": 256
+ },
+ {
+ "epoch": 2.29,
+ "grad_norm": 0.33515508602624905,
+ "learning_rate": 8.035878185042868e-05,
+ "loss": 0.8015,
+ "step": 257
+ },
+ {
+ "epoch": 2.3,
+ "grad_norm": 0.2905943721096322,
+ "learning_rate": 7.965439869473664e-05,
+ "loss": 0.8457,
+ "step": 258
+ },
+ {
+ "epoch": 2.31,
+ "grad_norm": 0.3140679719552616,
+ "learning_rate": 7.895106702997437e-05,
+ "loss": 0.8559,
+ "step": 259
+ },
+ {
+ "epoch": 2.31,
+ "grad_norm": 0.29745105018138573,
+ "learning_rate": 7.824882320536814e-05,
+ "loss": 0.7453,
+ "step": 260
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 0.29818631731197365,
+ "learning_rate": 7.754770351392311e-05,
+ "loss": 0.8354,
+ "step": 261
+ },
+ {
+ "epoch": 2.33,
+ "grad_norm": 0.24721488944366407,
+ "learning_rate": 7.684774419054747e-05,
+ "loss": 0.7755,
+ "step": 262
+ },
+ {
+ "epoch": 2.34,
+ "grad_norm": 0.31210442779019465,
+ "learning_rate": 7.614898141017996e-05,
+ "loss": 0.7208,
+ "step": 263
+ },
+ {
+ "epoch": 2.35,
+ "grad_norm": 0.2873220240109992,
+ "learning_rate": 7.54514512859201e-05,
+ "loss": 0.7548,
+ "step": 264
+ },
+ {
+ "epoch": 2.36,
+ "grad_norm": 0.3006634171776217,
+ "learning_rate": 7.475518986716194e-05,
+ "loss": 0.7566,
+ "step": 265
+ },
+ {
+ "epoch": 2.37,
+ "grad_norm": 0.2799417613336026,
+ "learning_rate": 7.406023313773097e-05,
+ "loss": 0.727,
+ "step": 266
+ },
+ {
+ "epoch": 2.38,
+ "grad_norm": 0.2451761866231664,
+ "learning_rate": 7.336661701402439e-05,
+ "loss": 0.9641,
+ "step": 267
+ },
+ {
+ "epoch": 2.39,
+ "grad_norm": 0.305202611125298,
+ "learning_rate": 7.267437734315492e-05,
+ "loss": 0.7891,
+ "step": 268
+ },
+ {
+ "epoch": 2.39,
+ "grad_norm": 0.29107717848747816,
+ "learning_rate": 7.198354990109805e-05,
+ "loss": 0.9032,
+ "step": 269
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 0.2688898665176787,
+ "learning_rate": 7.129417039084333e-05,
+ "loss": 0.8416,
+ "step": 270
+ },
+ {
+ "epoch": 2.41,
+ "grad_norm": 0.2814206029778395,
+ "learning_rate": 7.060627444054893e-05,
+ "loss": 0.8443,
+ "step": 271
+ },
+ {
+ "epoch": 2.42,
+ "grad_norm": 0.2862094867555512,
+ "learning_rate": 6.99198976017005e-05,
+ "loss": 0.8271,
+ "step": 272
+ },
+ {
+ "epoch": 2.43,
+ "grad_norm": 0.3214647340394826,
+ "learning_rate": 6.923507534727373e-05,
+ "loss": 0.7793,
+ "step": 273
+ },
+ {
+ "epoch": 2.44,
+ "grad_norm": 0.3033659714564417,
+ "learning_rate": 6.855184306990106e-05,
+ "loss": 0.7856,
+ "step": 274
+ },
+ {
+ "epoch": 2.45,
+ "grad_norm": 0.3024382342577774,
+ "learning_rate": 6.78702360800425e-05,
+ "loss": 0.8633,
+ "step": 275
+ },
+ {
+ "epoch": 2.46,
+ "grad_norm": 0.25803598196729505,
+ "learning_rate": 6.719028960416098e-05,
+ "loss": 0.8428,
+ "step": 276
+ },
+ {
+ "epoch": 2.47,
+ "grad_norm": 0.35469202971401803,
+ "learning_rate": 6.651203878290139e-05,
+ "loss": 0.8665,
+ "step": 277
+ },
+ {
+ "epoch": 2.47,
+ "grad_norm": 0.3122516837597691,
+ "learning_rate": 6.583551866927475e-05,
+ "loss": 0.8787,
+ "step": 278
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 0.3305470786367901,
+ "learning_rate": 6.516076422684654e-05,
+ "loss": 0.8765,
+ "step": 279
+ },
+ {
+ "epoch": 2.49,
+ "grad_norm": 0.3324622666488467,
+ "learning_rate": 6.448781032792972e-05,
+ "loss": 0.8318,
+ "step": 280
+ },
+ {
+ "epoch": 2.49,
+ "eval_loss": 1.2546111345291138,
+ "eval_runtime": 13.2379,
+ "eval_samples_per_second": 22.662,
+ "eval_steps_per_second": 2.871,
+ "step": 280
+ },
+ {
+ "epoch": 2.5,
+ "grad_norm": 0.342341713579355,
+ "learning_rate": 6.381669175178248e-05,
+ "loss": 0.9517,
+ "step": 281
+ },
+ {
+ "epoch": 2.51,
+ "grad_norm": 0.33913458352374665,
+ "learning_rate": 6.31474431828108e-05,
+ "loss": 0.8564,
+ "step": 282
+ },
+ {
+ "epoch": 2.52,
+ "grad_norm": 0.30528689383480295,
+ "learning_rate": 6.248009920877592e-05,
+ "loss": 0.8199,
+ "step": 283
+ },
+ {
+ "epoch": 2.53,
+ "grad_norm": 0.29698648367254743,
+ "learning_rate": 6.181469431900672e-05,
+ "loss": 0.785,
+ "step": 284
+ },
+ {
+ "epoch": 2.54,
+ "grad_norm": 0.32239262939282626,
+ "learning_rate": 6.115126290261745e-05,
+ "loss": 0.7794,
+ "step": 285
+ },
+ {
+ "epoch": 2.55,
+ "grad_norm": 0.2694595905080167,
+ "learning_rate": 6.048983924673022e-05,
+ "loss": 0.8056,
+ "step": 286
+ },
+ {
+ "epoch": 2.55,
+ "grad_norm": 0.3045496751154443,
+ "learning_rate": 5.983045753470308e-05,
+ "loss": 0.8164,
+ "step": 287
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 0.2927868214627918,
+ "learning_rate": 5.917315184436345e-05,
+ "loss": 0.8358,
+ "step": 288
+ },
+ {
+ "epoch": 2.57,
+ "grad_norm": 0.2931914055644858,
+ "learning_rate": 5.851795614624682e-05,
+ "loss": 0.8011,
+ "step": 289
+ },
+ {
+ "epoch": 2.58,
+ "grad_norm": 0.3158716819379082,
+ "learning_rate": 5.786490430184115e-05,
+ "loss": 0.8332,
+ "step": 290
+ },
+ {
+ "epoch": 2.59,
+ "grad_norm": 0.3482519147352008,
+ "learning_rate": 5.72140300618369e-05,
+ "loss": 0.7621,
+ "step": 291
+ },
+ {
+ "epoch": 2.6,
+ "grad_norm": 0.28652801822050894,
+ "learning_rate": 5.656536706438267e-05,
+ "loss": 0.77,
+ "step": 292
+ },
+ {
+ "epoch": 2.61,
+ "grad_norm": 0.29691290613407717,
+ "learning_rate": 5.591894883334667e-05,
+ "loss": 0.9394,
+ "step": 293
+ },
+ {
+ "epoch": 2.62,
+ "grad_norm": 0.26699581966985203,
+ "learning_rate": 5.5274808776584367e-05,
+ "loss": 0.7918,
+ "step": 294
+ },
+ {
+ "epoch": 2.63,
+ "grad_norm": 0.2926923719762685,
+ "learning_rate": 5.463298018421171e-05,
+ "loss": 0.8723,
+ "step": 295
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.3403087263187063,
+ "learning_rate": 5.399349622688479e-05,
+ "loss": 0.8097,
+ "step": 296
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.34261233464532476,
+ "learning_rate": 5.335638995408545e-05,
+ "loss": 0.9032,
+ "step": 297
+ },
+ {
+ "epoch": 2.65,
+ "grad_norm": 0.31315234759634086,
+ "learning_rate": 5.272169429241325e-05,
+ "loss": 0.82,
+ "step": 298
+ },
+ {
+ "epoch": 2.66,
+ "grad_norm": 0.3179759425444047,
+ "learning_rate": 5.208944204388377e-05,
+ "loss": 0.8864,
+ "step": 299
+ },
+ {
+ "epoch": 2.67,
+ "grad_norm": 0.3121296356843828,
+ "learning_rate": 5.145966588423341e-05,
+ "loss": 0.8258,
+ "step": 300
+ },
+ {
+ "epoch": 2.68,
+ "grad_norm": 0.268436849924173,
+ "learning_rate": 5.0832398361230596e-05,
+ "loss": 0.8906,
+ "step": 301
+ },
+ {
+ "epoch": 2.69,
+ "grad_norm": 0.2961161602467319,
+ "learning_rate": 5.020767189299369e-05,
+ "loss": 0.8828,
+ "step": 302
+ },
+ {
+ "epoch": 2.7,
+ "grad_norm": 0.27743957099992345,
+ "learning_rate": 4.9585518766315496e-05,
+ "loss": 0.8251,
+ "step": 303
+ },
+ {
+ "epoch": 2.71,
+ "grad_norm": 0.2949909861852426,
+ "learning_rate": 4.896597113499479e-05,
+ "loss": 0.7911,
+ "step": 304
+ },
+ {
+ "epoch": 2.72,
+ "grad_norm": 0.3161115451278363,
+ "learning_rate": 4.834906101817438e-05,
+ "loss": 0.8157,
+ "step": 305
+ },
+ {
+ "epoch": 2.72,
+ "grad_norm": 0.28720077046065867,
+ "learning_rate": 4.773482029868657e-05,
+ "loss": 0.82,
+ "step": 306
+ },
+ {
+ "epoch": 2.73,
+ "grad_norm": 0.4045319357608716,
+ "learning_rate": 4.712328072140505e-05,
+ "loss": 0.8414,
+ "step": 307
+ },
+ {
+ "epoch": 2.74,
+ "grad_norm": 0.3070232288390269,
+ "learning_rate": 4.651447389160458e-05,
+ "loss": 0.8427,
+ "step": 308
+ },
+ {
+ "epoch": 2.74,
+ "eval_loss": 1.2574400901794434,
+ "eval_runtime": 13.2473,
+ "eval_samples_per_second": 22.646,
+ "eval_steps_per_second": 2.869,
+ "step": 308
+ },
+ {
+ "epoch": 2.75,
+ "grad_norm": 0.3214782806968351,
+ "learning_rate": 4.5908431273327436e-05,
+ "loss": 0.8469,
+ "step": 309
+ },
+ {
+ "epoch": 2.76,
+ "grad_norm": 0.24241410698156174,
+ "learning_rate": 4.530518418775733e-05,
+ "loss": 0.8346,
+ "step": 310
+ },
+ {
+ "epoch": 2.77,
+ "grad_norm": 0.3303263594210879,
+ "learning_rate": 4.470476381160065e-05,
+ "loss": 0.8298,
+ "step": 311
+ },
+ {
+ "epoch": 2.78,
+ "grad_norm": 0.30711900849760865,
+ "learning_rate": 4.4107201175475275e-05,
+ "loss": 0.789,
+ "step": 312
+ },
+ {
+ "epoch": 2.79,
+ "grad_norm": 0.2954465859389713,
+ "learning_rate": 4.351252716230685e-05,
+ "loss": 0.8029,
+ "step": 313
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 0.29925087091531116,
+ "learning_rate": 4.292077250573266e-05,
+ "loss": 0.8633,
+ "step": 314
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 0.3177611223775825,
+ "learning_rate": 4.2331967788513295e-05,
+ "loss": 0.76,
+ "step": 315
+ },
+ {
+ "epoch": 2.81,
+ "grad_norm": 0.28642407848269513,
+ "learning_rate": 4.174614344095213e-05,
+ "loss": 0.823,
+ "step": 316
+ },
+ {
+ "epoch": 2.82,
+ "grad_norm": 0.3243224656005062,
+ "learning_rate": 4.116332973932256e-05,
+ "loss": 0.7831,
+ "step": 317
+ },
+ {
+ "epoch": 2.83,
+ "grad_norm": 0.34877334027822726,
+ "learning_rate": 4.058355680430337e-05,
+ "loss": 0.899,
+ "step": 318
+ },
+ {
+ "epoch": 2.84,
+ "grad_norm": 0.28640325479143114,
+ "learning_rate": 4.0006854599421926e-05,
+ "loss": 0.8292,
+ "step": 319
+ },
+ {
+ "epoch": 2.85,
+ "grad_norm": 0.3135316628014535,
+ "learning_rate": 3.943325292950579e-05,
+ "loss": 0.8731,
+ "step": 320
+ },
+ {
+ "epoch": 2.86,
+ "grad_norm": 0.2949970604257085,
+ "learning_rate": 3.886278143914219e-05,
+ "loss": 0.8402,
+ "step": 321
+ },
+ {
+ "epoch": 2.87,
+ "grad_norm": 0.30057896586780075,
+ "learning_rate": 3.829546961114607e-05,
+ "loss": 0.7713,
+ "step": 322
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 0.3558574270285126,
+ "learning_rate": 3.773134676503629e-05,
+ "loss": 0.8435,
+ "step": 323
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 0.29115288332943334,
+ "learning_rate": 3.7170442055520415e-05,
+ "loss": 0.9022,
+ "step": 324
+ },
+ {
+ "epoch": 2.89,
+ "grad_norm": 0.3192074718527619,
+ "learning_rate": 3.661278447098789e-05,
+ "loss": 0.7662,
+ "step": 325
+ },
+ {
+ "epoch": 2.9,
+ "grad_norm": 0.33335742888185405,
+ "learning_rate": 3.605840283201195e-05,
+ "loss": 0.8111,
+ "step": 326
+ },
+ {
+ "epoch": 2.91,
+ "grad_norm": 0.29748212071395186,
+ "learning_rate": 3.550732578986006e-05,
+ "loss": 0.7543,
+ "step": 327
+ },
+ {
+ "epoch": 2.92,
+ "grad_norm": 0.3680409192627914,
+ "learning_rate": 3.495958182501325e-05,
+ "loss": 0.8124,
+ "step": 328
+ },
+ {
+ "epoch": 2.93,
+ "grad_norm": 0.27807302364345643,
+ "learning_rate": 3.441519924569408e-05,
+ "loss": 0.7856,
+ "step": 329
+ },
+ {
+ "epoch": 2.94,
+ "grad_norm": 0.3050855823733691,
+ "learning_rate": 3.387420618640379e-05,
+ "loss": 0.8506,
+ "step": 330
+ },
+ {
+ "epoch": 2.95,
+ "grad_norm": 0.3322620263029238,
+ "learning_rate": 3.3336630606468134e-05,
+ "loss": 0.8771,
+ "step": 331
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 0.3112008867427982,
+ "learning_rate": 3.280250028859248e-05,
+ "loss": 0.7785,
+ "step": 332
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 0.2839548329095365,
+ "learning_rate": 3.227184283742591e-05,
+ "loss": 0.9153,
+ "step": 333
+ },
+ {
+ "epoch": 2.97,
+ "grad_norm": 0.34615397822650606,
+ "learning_rate": 3.174468567813461e-05,
+ "loss": 0.7753,
+ "step": 334
+ },
+ {
+ "epoch": 2.98,
+ "grad_norm": 0.34691866307772695,
+ "learning_rate": 3.122105605498442e-05,
+ "loss": 0.851,
+ "step": 335
+ },
+ {
+ "epoch": 2.99,
+ "grad_norm": 0.296369624391198,
+ "learning_rate": 3.070098102993302e-05,
+ "loss": 0.8572,
+ "step": 336
+ },
+ {
+ "epoch": 2.99,
+ "eval_loss": 1.2511259317398071,
+ "eval_runtime": 13.2202,
+ "eval_samples_per_second": 22.692,
+ "eval_steps_per_second": 2.874,
+ "step": 336
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 448,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 4,
+ "save_steps": 112,
+ "total_flos": 4.005448726012232e+17,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-336/training_args.bin b/checkpoint-336/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4992a3b445f8065303289f8a30d4fedc43285059
--- /dev/null
+++ b/checkpoint-336/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d400c16f982c36b10268ff7e69e878c44d11f5fb692a61770a8e1efb50d4491c
+size 6776
diff --git a/checkpoint-336/zero_to_fp32.py b/checkpoint-336/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..49b846633d6eb1e836e34681e44033581f4edb7b
--- /dev/null
+++ b/checkpoint-336/zero_to_fp32.py
@@ -0,0 +1,592 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+ total_files = len(files)
+ state_dicts = []
+ for f in files:
+ state_dict = torch.load(f, map_location=device)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ if zero_stage <= 2:
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ elif zero_stage == 3:
+ # if there is more than one param group, there will be multiple flattened tensors - one
+ # flattened tensor per group - for simplicity merge them into a single tensor
+ #
+ # XXX: could make the script more memory efficient for when there are multiple groups - it
+ # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+ fp32_flat_groups = [
+ torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+ ]
+
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ model_files = get_model_state_files(ds_checkpoint_dir)
+
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ for name, shape in param_shapes.items():
+
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # XXX: memory usage doubles here
+ state_dict[name] = torch.cat(
+ tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+ 0).narrow(0, 0, unpartitioned_numel).view(shape)
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+ you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ """
+
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+ print(f"Saving fp32 state dict to {output_file}")
+ torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info(f"Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info(f"Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument(
+ "output_file",
+ type=str,
+ help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)
diff --git a/checkpoint-372/README.md b/checkpoint-372/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..cc48d049490be40c7b7654441aabee7b5bc360d2
--- /dev/null
+++ b/checkpoint-372/README.md
@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: google/gemma-7b-it
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.9.0
\ No newline at end of file
diff --git a/checkpoint-372/adapter_config.json b/checkpoint-372/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c3e2768efa37865b99706584c01866eafc6c45a
--- /dev/null
+++ b/checkpoint-372/adapter_config.json
@@ -0,0 +1,33 @@
+{
+ "alpha_pattern": {},
+ "auto_mapping": null,
+ "base_model_name_or_path": "google/gemma-7b-it",
+ "bias": "none",
+ "fan_in_fan_out": null,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "o_proj",
+ "k_proj",
+ "q_proj",
+ "gate_proj",
+ "up_proj",
+ "v_proj"
+ ],
+ "task_type": "CAUSAL_LM",
+ "use_dora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-372/adapter_model.safetensors b/checkpoint-372/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0015bf12d8aa10457cc3eafcef7b9515dd2b63c3
--- /dev/null
+++ b/checkpoint-372/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c23b965687f7bf2e033e1e8051de69e24c99f3103c06606007e68485ebfabea
+size 200068904
diff --git a/checkpoint-372/global_step372/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-372/global_step372/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a2f184187d5fb713c7f812f3d447fc319771256
--- /dev/null
+++ b/checkpoint-372/global_step372/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712292fc7e5a6d570c1376cc3be7e12dab2d34fb7ffe48281da38c8053603a39
+size 150126608
diff --git a/checkpoint-372/global_step372/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-372/global_step372/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c45e01f7296a4d841e594606caacd3859dcfd8d6
--- /dev/null
+++ b/checkpoint-372/global_step372/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3c6bbd461df3af80fb33496e1907ef542102bfa96434d50c174fe80c0dd98e4
+size 150126672
diff --git a/checkpoint-372/global_step372/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/checkpoint-372/global_step372/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76e00cc0290562adc1b6b707d91f81a3ae2a6b4f
--- /dev/null
+++ b/checkpoint-372/global_step372/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46fee0787345b6483d7a54f3ceeb3260a7a8bef008c22e24c18225027433ff01
+size 150126736
diff --git a/checkpoint-372/global_step372/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/checkpoint-372/global_step372/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd1c04607ffa6bcedba73d2be3596c254a85645d
--- /dev/null
+++ b/checkpoint-372/global_step372/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fbe9084b027f2164f5fa8039ea7d37a722d0e0f9f70b2a76fa605e462a2ad6e
+size 150126736
diff --git a/checkpoint-372/global_step372/mp_rank_00_model_states.pt b/checkpoint-372/global_step372/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..179e333ae5d1502ea68592df6a33c543b58d2f27
--- /dev/null
+++ b/checkpoint-372/global_step372/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a38ea9a669e473fff57e6c134dd6703ddacc9f123121be165e81bcdcad09513b
+size 1896781286
diff --git a/checkpoint-372/latest b/checkpoint-372/latest
new file mode 100644
index 0000000000000000000000000000000000000000..58718163630dae3a397bab9fcebd1e5516bfae6b
--- /dev/null
+++ b/checkpoint-372/latest
@@ -0,0 +1 @@
+global_step372
\ No newline at end of file
diff --git a/checkpoint-372/rng_state_0.pth b/checkpoint-372/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..ecbda9fe0809770d12955f1732c0114453213ea4
--- /dev/null
+++ b/checkpoint-372/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ebe22192845fac896cd970f52665ebcfd6b5796077804b55f0d8830fcfa32be
+size 15024
diff --git a/checkpoint-372/rng_state_1.pth b/checkpoint-372/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..3636f8e8c132361277c40a90a327d9a54d5cd508
--- /dev/null
+++ b/checkpoint-372/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5bbd2194b05d2155d794f7732bdab8deaa38ee92f4c49fa250d0c9f0fd5f532
+size 15024
diff --git a/checkpoint-372/rng_state_2.pth b/checkpoint-372/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f480bc42cd80af8138987ef5eabe99d53ee6b554
--- /dev/null
+++ b/checkpoint-372/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f480e768d1b3f6474c222ba1e9d373d3fa99aeb3a944de3d1648ac20b4077d2a
+size 15024
diff --git a/checkpoint-372/rng_state_3.pth b/checkpoint-372/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..59cbcb27ed2a9c4f965cc32825b10cb072d551b0
--- /dev/null
+++ b/checkpoint-372/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58480874185eed88dd61379fe4c13e95c8e8899caf976ff7beca6c2c29f825de
+size 15024
diff --git a/checkpoint-372/scheduler.pt b/checkpoint-372/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6a29ccdd842ef5dc3c8960bad30268d73a2dc3af
--- /dev/null
+++ b/checkpoint-372/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a1b84ad14ee5a7b2967e455e771394bef96a10da0abeba0c7fac61961ff2bf
+size 1064
diff --git a/checkpoint-372/trainer_state.json b/checkpoint-372/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..e74134b082788735a316b3b09e64d90837cf20fb
--- /dev/null
+++ b/checkpoint-372/trainer_state.json
@@ -0,0 +1,2753 @@
+{
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 3.9715302491103204,
+ "eval_steps": 24,
+ "global_step": 372,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.01,
+ "grad_norm": 1.8206765789002874,
+ "learning_rate": 2.2222222222222223e-05,
+ "loss": 5.0474,
+ "step": 1
+ },
+ {
+ "epoch": 0.01,
+ "eval_loss": 5.927858829498291,
+ "eval_runtime": 117.3665,
+ "eval_samples_per_second": 8.512,
+ "eval_steps_per_second": 0.537,
+ "step": 1
+ },
+ {
+ "epoch": 0.02,
+ "grad_norm": 1.9889295079554647,
+ "learning_rate": 4.4444444444444447e-05,
+ "loss": 5.5569,
+ "step": 2
+ },
+ {
+ "epoch": 0.03,
+ "grad_norm": 1.8931443004310682,
+ "learning_rate": 6.666666666666667e-05,
+ "loss": 5.2383,
+ "step": 3
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 2.195266234429632,
+ "learning_rate": 8.888888888888889e-05,
+ "loss": 5.4943,
+ "step": 4
+ },
+ {
+ "epoch": 0.05,
+ "grad_norm": 2.6001064132041503,
+ "learning_rate": 0.00011111111111111112,
+ "loss": 5.2602,
+ "step": 5
+ },
+ {
+ "epoch": 0.06,
+ "grad_norm": 3.26301463076567,
+ "learning_rate": 0.00013333333333333334,
+ "loss": 4.8182,
+ "step": 6
+ },
+ {
+ "epoch": 0.07,
+ "grad_norm": 3.476044691292363,
+ "learning_rate": 0.00015555555555555556,
+ "loss": 4.0432,
+ "step": 7
+ },
+ {
+ "epoch": 0.09,
+ "grad_norm": 3.378803229553045,
+ "learning_rate": 0.00017777777777777779,
+ "loss": 3.5212,
+ "step": 8
+ },
+ {
+ "epoch": 0.1,
+ "grad_norm": 3.9419449437137017,
+ "learning_rate": 0.0002,
+ "loss": 3.2239,
+ "step": 9
+ },
+ {
+ "epoch": 0.11,
+ "grad_norm": 5.8833082175146485,
+ "learning_rate": 0.00019999625498303932,
+ "loss": 3.4319,
+ "step": 10
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 5.4690223843996515,
+ "learning_rate": 0.0001999850202126604,
+ "loss": 2.8167,
+ "step": 11
+ },
+ {
+ "epoch": 0.13,
+ "grad_norm": 7.009614336449043,
+ "learning_rate": 0.00019996629653035126,
+ "loss": 2.7966,
+ "step": 12
+ },
+ {
+ "epoch": 0.14,
+ "grad_norm": 6.254841874500106,
+ "learning_rate": 0.0001999400853385221,
+ "loss": 2.1336,
+ "step": 13
+ },
+ {
+ "epoch": 0.15,
+ "grad_norm": 6.037710889841169,
+ "learning_rate": 0.00019990638860040006,
+ "loss": 1.85,
+ "step": 14
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0500019118881985,
+ "learning_rate": 0.00019986520883988232,
+ "loss": 1.5964,
+ "step": 15
+ },
+ {
+ "epoch": 0.17,
+ "grad_norm": 0.6169710624824223,
+ "learning_rate": 0.00019981654914134686,
+ "loss": 1.4307,
+ "step": 16
+ },
+ {
+ "epoch": 0.18,
+ "grad_norm": 1.86114059095932,
+ "learning_rate": 0.00019976041314942155,
+ "loss": 1.4285,
+ "step": 17
+ },
+ {
+ "epoch": 0.19,
+ "grad_norm": 1.6513877610200167,
+ "learning_rate": 0.00019969680506871137,
+ "loss": 1.4621,
+ "step": 18
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.4395882738454628,
+ "learning_rate": 0.000199625729663483,
+ "loss": 1.3561,
+ "step": 19
+ },
+ {
+ "epoch": 0.21,
+ "grad_norm": 0.70847060238536,
+ "learning_rate": 0.00019954719225730847,
+ "loss": 1.3565,
+ "step": 20
+ },
+ {
+ "epoch": 0.22,
+ "grad_norm": 0.4331630595385925,
+ "learning_rate": 0.00019946119873266613,
+ "loss": 1.3374,
+ "step": 21
+ },
+ {
+ "epoch": 0.23,
+ "grad_norm": 0.5580281682185451,
+ "learning_rate": 0.0001993677555305002,
+ "loss": 1.313,
+ "step": 22
+ },
+ {
+ "epoch": 0.25,
+ "grad_norm": 0.5217443953771937,
+ "learning_rate": 0.00019926686964973813,
+ "loss": 1.2541,
+ "step": 23
+ },
+ {
+ "epoch": 0.26,
+ "grad_norm": 0.36823120314463453,
+ "learning_rate": 0.00019915854864676664,
+ "loss": 1.2191,
+ "step": 24
+ },
+ {
+ "epoch": 0.26,
+ "eval_loss": 1.2946609258651733,
+ "eval_runtime": 118.9039,
+ "eval_samples_per_second": 8.402,
+ "eval_steps_per_second": 0.53,
+ "step": 24
+ },
+ {
+ "epoch": 0.27,
+ "grad_norm": 0.5797477063688413,
+ "learning_rate": 0.0001990428006348656,
+ "loss": 1.24,
+ "step": 25
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 0.41369538857234545,
+ "learning_rate": 0.00019891963428360043,
+ "loss": 1.209,
+ "step": 26
+ },
+ {
+ "epoch": 0.29,
+ "grad_norm": 0.36666008426797836,
+ "learning_rate": 0.00019878905881817252,
+ "loss": 1.2543,
+ "step": 27
+ },
+ {
+ "epoch": 0.3,
+ "grad_norm": 0.3976779691989045,
+ "learning_rate": 0.00019865108401872857,
+ "loss": 1.2431,
+ "step": 28
+ },
+ {
+ "epoch": 0.31,
+ "grad_norm": 0.4992861718630414,
+ "learning_rate": 0.00019850572021962788,
+ "loss": 1.2471,
+ "step": 29
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.33729072192890136,
+ "learning_rate": 0.00019835297830866826,
+ "loss": 1.1933,
+ "step": 30
+ },
+ {
+ "epoch": 0.33,
+ "grad_norm": 0.29373457949318904,
+ "learning_rate": 0.00019819286972627066,
+ "loss": 1.1761,
+ "step": 31
+ },
+ {
+ "epoch": 0.34,
+ "grad_norm": 0.5339184947140588,
+ "learning_rate": 0.0001980254064646223,
+ "loss": 1.165,
+ "step": 32
+ },
+ {
+ "epoch": 0.35,
+ "grad_norm": 0.38755069216510263,
+ "learning_rate": 0.00019785060106677818,
+ "loss": 1.1236,
+ "step": 33
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 0.338373181403367,
+ "learning_rate": 0.00019766846662572191,
+ "loss": 1.2102,
+ "step": 34
+ },
+ {
+ "epoch": 0.37,
+ "grad_norm": 0.39237714718744304,
+ "learning_rate": 0.00019747901678338496,
+ "loss": 1.1642,
+ "step": 35
+ },
+ {
+ "epoch": 0.38,
+ "grad_norm": 0.3614249847081747,
+ "learning_rate": 0.00019728226572962473,
+ "loss": 1.1387,
+ "step": 36
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.28278007479509987,
+ "learning_rate": 0.00019707822820116193,
+ "loss": 1.0939,
+ "step": 37
+ },
+ {
+ "epoch": 0.41,
+ "grad_norm": 0.3008254873268798,
+ "learning_rate": 0.00019686691948047664,
+ "loss": 1.1346,
+ "step": 38
+ },
+ {
+ "epoch": 0.42,
+ "grad_norm": 0.4263010439416343,
+ "learning_rate": 0.0001966483553946637,
+ "loss": 1.1015,
+ "step": 39
+ },
+ {
+ "epoch": 0.43,
+ "grad_norm": 0.32725448028464205,
+ "learning_rate": 0.00019642255231424729,
+ "loss": 1.1324,
+ "step": 40
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 0.3028242900588441,
+ "learning_rate": 0.00019618952715195475,
+ "loss": 1.1147,
+ "step": 41
+ },
+ {
+ "epoch": 0.45,
+ "grad_norm": 0.33893311928252234,
+ "learning_rate": 0.00019594929736144976,
+ "loss": 1.0978,
+ "step": 42
+ },
+ {
+ "epoch": 0.46,
+ "grad_norm": 0.2786082334492372,
+ "learning_rate": 0.0001957018809360251,
+ "loss": 1.0933,
+ "step": 43
+ },
+ {
+ "epoch": 0.47,
+ "grad_norm": 0.2732185168098956,
+ "learning_rate": 0.00019544729640725498,
+ "loss": 1.084,
+ "step": 44
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.33386436894143035,
+ "learning_rate": 0.00019518556284360696,
+ "loss": 1.0673,
+ "step": 45
+ },
+ {
+ "epoch": 0.49,
+ "grad_norm": 0.2761688734050621,
+ "learning_rate": 0.00019491669984901379,
+ "loss": 1.0523,
+ "step": 46
+ },
+ {
+ "epoch": 0.5,
+ "grad_norm": 0.3346957388610895,
+ "learning_rate": 0.00019464072756140486,
+ "loss": 1.0913,
+ "step": 47
+ },
+ {
+ "epoch": 0.51,
+ "grad_norm": 0.30196058996924285,
+ "learning_rate": 0.0001943576666511982,
+ "loss": 1.1165,
+ "step": 48
+ },
+ {
+ "epoch": 0.51,
+ "eval_loss": 1.167867660522461,
+ "eval_runtime": 119.1485,
+ "eval_samples_per_second": 8.384,
+ "eval_steps_per_second": 0.529,
+ "step": 48
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 0.27445390350987153,
+ "learning_rate": 0.00019406753831975203,
+ "loss": 1.1069,
+ "step": 49
+ },
+ {
+ "epoch": 0.53,
+ "grad_norm": 0.34729097228771255,
+ "learning_rate": 0.00019377036429777672,
+ "loss": 1.0567,
+ "step": 50
+ },
+ {
+ "epoch": 0.54,
+ "grad_norm": 0.31314016575739406,
+ "learning_rate": 0.0001934661668437073,
+ "loss": 1.0875,
+ "step": 51
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.29140014335226905,
+ "learning_rate": 0.0001931549687420364,
+ "loss": 1.0929,
+ "step": 52
+ },
+ {
+ "epoch": 0.57,
+ "grad_norm": 0.2638104110161505,
+ "learning_rate": 0.00019283679330160726,
+ "loss": 1.0963,
+ "step": 53
+ },
+ {
+ "epoch": 0.58,
+ "grad_norm": 0.2833945318119855,
+ "learning_rate": 0.0001925116643538684,
+ "loss": 1.0535,
+ "step": 54
+ },
+ {
+ "epoch": 0.59,
+ "grad_norm": 0.28672689795285417,
+ "learning_rate": 0.0001921796062510882,
+ "loss": 1.0699,
+ "step": 55
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 0.261255409262294,
+ "learning_rate": 0.00019184064386453128,
+ "loss": 1.0658,
+ "step": 56
+ },
+ {
+ "epoch": 0.61,
+ "grad_norm": 0.24304864434604007,
+ "learning_rate": 0.00019149480258259533,
+ "loss": 1.0441,
+ "step": 57
+ },
+ {
+ "epoch": 0.62,
+ "grad_norm": 0.2987107937915846,
+ "learning_rate": 0.00019114210830890969,
+ "loss": 1.0061,
+ "step": 58
+ },
+ {
+ "epoch": 0.63,
+ "grad_norm": 0.2617045441373282,
+ "learning_rate": 0.00019078258746039507,
+ "loss": 1.0578,
+ "step": 59
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.2577955355987167,
+ "learning_rate": 0.00019041626696528503,
+ "loss": 1.0333,
+ "step": 60
+ },
+ {
+ "epoch": 0.65,
+ "grad_norm": 0.2823058812174375,
+ "learning_rate": 0.0001900431742611089,
+ "loss": 1.0837,
+ "step": 61
+ },
+ {
+ "epoch": 0.66,
+ "grad_norm": 0.30425238718712166,
+ "learning_rate": 0.00018966333729263674,
+ "loss": 1.0619,
+ "step": 62
+ },
+ {
+ "epoch": 0.67,
+ "grad_norm": 0.29826831116146957,
+ "learning_rate": 0.0001892767845097864,
+ "loss": 1.056,
+ "step": 63
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 0.22990267950533677,
+ "learning_rate": 0.00018888354486549237,
+ "loss": 1.061,
+ "step": 64
+ },
+ {
+ "epoch": 0.69,
+ "grad_norm": 0.27604852373975236,
+ "learning_rate": 0.00018848364781353744,
+ "loss": 1.0624,
+ "step": 65
+ },
+ {
+ "epoch": 0.7,
+ "grad_norm": 0.302101014156969,
+ "learning_rate": 0.00018807712330634642,
+ "loss": 1.0965,
+ "step": 66
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.2532153192142023,
+ "learning_rate": 0.00018766400179274286,
+ "loss": 1.0972,
+ "step": 67
+ },
+ {
+ "epoch": 0.73,
+ "grad_norm": 0.23803088057755897,
+ "learning_rate": 0.00018724431421566823,
+ "loss": 1.0823,
+ "step": 68
+ },
+ {
+ "epoch": 0.74,
+ "grad_norm": 0.2200041903156331,
+ "learning_rate": 0.0001868180920098644,
+ "loss": 1.037,
+ "step": 69
+ },
+ {
+ "epoch": 0.75,
+ "grad_norm": 0.31123761066229655,
+ "learning_rate": 0.00018638536709951917,
+ "loss": 1.0689,
+ "step": 70
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 0.2760757149384919,
+ "learning_rate": 0.00018594617189587512,
+ "loss": 1.0071,
+ "step": 71
+ },
+ {
+ "epoch": 0.77,
+ "grad_norm": 0.2452672521810973,
+ "learning_rate": 0.00018550053929480202,
+ "loss": 1.0711,
+ "step": 72
+ },
+ {
+ "epoch": 0.77,
+ "eval_loss": 1.1377497911453247,
+ "eval_runtime": 119.461,
+ "eval_samples_per_second": 8.363,
+ "eval_steps_per_second": 0.527,
+ "step": 72
+ },
+ {
+ "epoch": 0.78,
+ "grad_norm": 0.30897216290479246,
+ "learning_rate": 0.0001850485026743328,
+ "loss": 1.0508,
+ "step": 73
+ },
+ {
+ "epoch": 0.79,
+ "grad_norm": 0.24165903393157925,
+ "learning_rate": 0.00018459009589216364,
+ "loss": 1.046,
+ "step": 74
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.2509819208307879,
+ "learning_rate": 0.00018412535328311814,
+ "loss": 1.0726,
+ "step": 75
+ },
+ {
+ "epoch": 0.81,
+ "grad_norm": 0.26145395006758515,
+ "learning_rate": 0.00018365430965657526,
+ "loss": 0.9998,
+ "step": 76
+ },
+ {
+ "epoch": 0.82,
+ "grad_norm": 0.26920709605794424,
+ "learning_rate": 0.00018317700029386245,
+ "loss": 1.065,
+ "step": 77
+ },
+ {
+ "epoch": 0.83,
+ "grad_norm": 0.24226754926786417,
+ "learning_rate": 0.0001826934609456129,
+ "loss": 1.0489,
+ "step": 78
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 0.3022365661006827,
+ "learning_rate": 0.00018220372782908777,
+ "loss": 1.0372,
+ "step": 79
+ },
+ {
+ "epoch": 0.85,
+ "grad_norm": 0.25795710005352673,
+ "learning_rate": 0.00018170783762546365,
+ "loss": 1.0128,
+ "step": 80
+ },
+ {
+ "epoch": 0.86,
+ "grad_norm": 0.3490748875058354,
+ "learning_rate": 0.00018120582747708502,
+ "loss": 1.0168,
+ "step": 81
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.24938209735120945,
+ "learning_rate": 0.00018069773498468223,
+ "loss": 0.9586,
+ "step": 82
+ },
+ {
+ "epoch": 0.89,
+ "grad_norm": 0.2527612545099894,
+ "learning_rate": 0.00018018359820455536,
+ "loss": 1.0385,
+ "step": 83
+ },
+ {
+ "epoch": 0.9,
+ "grad_norm": 0.27528879975094916,
+ "learning_rate": 0.0001796634556457236,
+ "loss": 1.0328,
+ "step": 84
+ },
+ {
+ "epoch": 0.91,
+ "grad_norm": 0.2605002777661913,
+ "learning_rate": 0.0001791373462670411,
+ "loss": 0.9966,
+ "step": 85
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 0.3117107796665858,
+ "learning_rate": 0.00017860530947427875,
+ "loss": 0.9772,
+ "step": 86
+ },
+ {
+ "epoch": 0.93,
+ "grad_norm": 0.28336227154677734,
+ "learning_rate": 0.0001780673851171728,
+ "loss": 1.0724,
+ "step": 87
+ },
+ {
+ "epoch": 0.94,
+ "grad_norm": 0.42707817919652674,
+ "learning_rate": 0.0001775236134864401,
+ "loss": 1.0038,
+ "step": 88
+ },
+ {
+ "epoch": 0.95,
+ "grad_norm": 0.29236016959846456,
+ "learning_rate": 0.0001769740353107602,
+ "loss": 1.0083,
+ "step": 89
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 0.43295063403530637,
+ "learning_rate": 0.00017641869175372493,
+ "loss": 1.022,
+ "step": 90
+ },
+ {
+ "epoch": 0.97,
+ "grad_norm": 0.3086663897043129,
+ "learning_rate": 0.00017585762441075503,
+ "loss": 1.0303,
+ "step": 91
+ },
+ {
+ "epoch": 0.98,
+ "grad_norm": 0.2783768981163154,
+ "learning_rate": 0.0001752908753059849,
+ "loss": 1.061,
+ "step": 92
+ },
+ {
+ "epoch": 0.99,
+ "grad_norm": 0.43168501819843275,
+ "learning_rate": 0.00017471848688911464,
+ "loss": 1.0631,
+ "step": 93
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 0.25487494913299935,
+ "learning_rate": 0.0001741405020322309,
+ "loss": 0.9858,
+ "step": 94
+ },
+ {
+ "epoch": 1.01,
+ "grad_norm": 0.3229761094582219,
+ "learning_rate": 0.00017355696402659548,
+ "loss": 0.9495,
+ "step": 95
+ },
+ {
+ "epoch": 1.02,
+ "grad_norm": 0.3178464701266748,
+ "learning_rate": 0.000172967916579403,
+ "loss": 0.9546,
+ "step": 96
+ },
+ {
+ "epoch": 1.02,
+ "eval_loss": 1.1303094625473022,
+ "eval_runtime": 119.6761,
+ "eval_samples_per_second": 8.348,
+ "eval_steps_per_second": 0.526,
+ "step": 96
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.2534616980189548,
+ "learning_rate": 0.00017237340381050703,
+ "loss": 0.9509,
+ "step": 97
+ },
+ {
+ "epoch": 1.05,
+ "grad_norm": 0.2354382873554396,
+ "learning_rate": 0.00017177347024911562,
+ "loss": 0.9611,
+ "step": 98
+ },
+ {
+ "epoch": 1.06,
+ "grad_norm": 0.2754259154521738,
+ "learning_rate": 0.00017116816083045602,
+ "loss": 0.9184,
+ "step": 99
+ },
+ {
+ "epoch": 1.07,
+ "grad_norm": 0.25868181129480755,
+ "learning_rate": 0.00017055752089240907,
+ "loss": 0.957,
+ "step": 100
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 0.2383943586330267,
+ "learning_rate": 0.00016994159617211317,
+ "loss": 0.9638,
+ "step": 101
+ },
+ {
+ "epoch": 1.09,
+ "grad_norm": 0.2706420372628291,
+ "learning_rate": 0.0001693204328025389,
+ "loss": 0.9115,
+ "step": 102
+ },
+ {
+ "epoch": 1.1,
+ "grad_norm": 0.2751042656041904,
+ "learning_rate": 0.0001686940773090333,
+ "loss": 0.9277,
+ "step": 103
+ },
+ {
+ "epoch": 1.11,
+ "grad_norm": 0.27700872737428867,
+ "learning_rate": 0.00016806257660583534,
+ "loss": 0.9248,
+ "step": 104
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.3350046312844708,
+ "learning_rate": 0.00016742597799256182,
+ "loss": 0.928,
+ "step": 105
+ },
+ {
+ "epoch": 1.13,
+ "grad_norm": 0.4055944986440079,
+ "learning_rate": 0.00016678432915066488,
+ "loss": 0.9074,
+ "step": 106
+ },
+ {
+ "epoch": 1.14,
+ "grad_norm": 0.2515177402600531,
+ "learning_rate": 0.00016613767813986044,
+ "loss": 0.9564,
+ "step": 107
+ },
+ {
+ "epoch": 1.15,
+ "grad_norm": 0.2571149695502646,
+ "learning_rate": 0.00016548607339452853,
+ "loss": 0.93,
+ "step": 108
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 0.38608942941048996,
+ "learning_rate": 0.0001648295637200856,
+ "loss": 0.9281,
+ "step": 109
+ },
+ {
+ "epoch": 1.17,
+ "grad_norm": 0.31939838976976676,
+ "learning_rate": 0.000164168198289329,
+ "loss": 0.9914,
+ "step": 110
+ },
+ {
+ "epoch": 1.19,
+ "grad_norm": 0.30504937567650897,
+ "learning_rate": 0.00016350202663875386,
+ "loss": 0.9549,
+ "step": 111
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.3320388344291162,
+ "learning_rate": 0.0001628310986648427,
+ "loss": 0.9086,
+ "step": 112
+ },
+ {
+ "epoch": 1.21,
+ "grad_norm": 0.27715569151296165,
+ "learning_rate": 0.0001621554646203284,
+ "loss": 0.8537,
+ "step": 113
+ },
+ {
+ "epoch": 1.22,
+ "grad_norm": 0.278787508566418,
+ "learning_rate": 0.0001614751751104301,
+ "loss": 0.9354,
+ "step": 114
+ },
+ {
+ "epoch": 1.23,
+ "grad_norm": 0.24483614460003267,
+ "learning_rate": 0.00016079028108906282,
+ "loss": 0.8996,
+ "step": 115
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 0.37520609596400134,
+ "learning_rate": 0.0001601008338550211,
+ "loss": 0.9514,
+ "step": 116
+ },
+ {
+ "epoch": 1.25,
+ "grad_norm": 0.2565631505653599,
+ "learning_rate": 0.00015940688504813662,
+ "loss": 0.8984,
+ "step": 117
+ },
+ {
+ "epoch": 1.26,
+ "grad_norm": 0.26348552476529935,
+ "learning_rate": 0.00015870848664541044,
+ "loss": 0.8941,
+ "step": 118
+ },
+ {
+ "epoch": 1.27,
+ "grad_norm": 0.32431198985496534,
+ "learning_rate": 0.00015800569095711982,
+ "loss": 0.8876,
+ "step": 119
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 0.29308039763069227,
+ "learning_rate": 0.00015729855062290022,
+ "loss": 0.9309,
+ "step": 120
+ },
+ {
+ "epoch": 1.28,
+ "eval_loss": 1.129751205444336,
+ "eval_runtime": 119.1497,
+ "eval_samples_per_second": 8.384,
+ "eval_steps_per_second": 0.529,
+ "step": 120
+ },
+ {
+ "epoch": 1.29,
+ "grad_norm": 0.2793291380060977,
+ "learning_rate": 0.0001565871186078025,
+ "loss": 0.9453,
+ "step": 121
+ },
+ {
+ "epoch": 1.3,
+ "grad_norm": 0.28873644301555734,
+ "learning_rate": 0.000155871448198326,
+ "loss": 0.9243,
+ "step": 122
+ },
+ {
+ "epoch": 1.31,
+ "grad_norm": 0.3086103724578039,
+ "learning_rate": 0.00015515159299842707,
+ "loss": 0.8877,
+ "step": 123
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 0.30407892484693505,
+ "learning_rate": 0.00015442760692550443,
+ "loss": 0.9448,
+ "step": 124
+ },
+ {
+ "epoch": 1.33,
+ "grad_norm": 0.29771602861368474,
+ "learning_rate": 0.00015369954420636048,
+ "loss": 0.889,
+ "step": 125
+ },
+ {
+ "epoch": 1.35,
+ "grad_norm": 0.30480490158838136,
+ "learning_rate": 0.00015296745937313987,
+ "loss": 0.9405,
+ "step": 126
+ },
+ {
+ "epoch": 1.36,
+ "grad_norm": 0.2949192855418127,
+ "learning_rate": 0.00015223140725924495,
+ "loss": 0.9382,
+ "step": 127
+ },
+ {
+ "epoch": 1.37,
+ "grad_norm": 0.2813631863132807,
+ "learning_rate": 0.00015149144299522873,
+ "loss": 0.9526,
+ "step": 128
+ },
+ {
+ "epoch": 1.38,
+ "grad_norm": 0.28548924064070513,
+ "learning_rate": 0.00015074762200466556,
+ "loss": 0.9174,
+ "step": 129
+ },
+ {
+ "epoch": 1.39,
+ "grad_norm": 0.28137053449960464,
+ "learning_rate": 0.00015000000000000001,
+ "loss": 0.9244,
+ "step": 130
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 0.2626750895717777,
+ "learning_rate": 0.00014924863297837378,
+ "loss": 0.9335,
+ "step": 131
+ },
+ {
+ "epoch": 1.41,
+ "grad_norm": 0.26686502371015536,
+ "learning_rate": 0.00014849357721743168,
+ "loss": 0.8948,
+ "step": 132
+ },
+ {
+ "epoch": 1.42,
+ "grad_norm": 0.3332273481179679,
+ "learning_rate": 0.00014773488927110633,
+ "loss": 0.9274,
+ "step": 133
+ },
+ {
+ "epoch": 1.43,
+ "grad_norm": 0.2528048763375234,
+ "learning_rate": 0.00014697262596538227,
+ "loss": 0.8731,
+ "step": 134
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.27184211707488076,
+ "learning_rate": 0.00014620684439403962,
+ "loss": 0.9318,
+ "step": 135
+ },
+ {
+ "epoch": 1.45,
+ "grad_norm": 0.3051111137538683,
+ "learning_rate": 0.0001454376019143779,
+ "loss": 0.9447,
+ "step": 136
+ },
+ {
+ "epoch": 1.46,
+ "grad_norm": 0.28771401659835155,
+ "learning_rate": 0.00014466495614291977,
+ "loss": 0.9343,
+ "step": 137
+ },
+ {
+ "epoch": 1.47,
+ "grad_norm": 0.28995797921621524,
+ "learning_rate": 0.0001438889649510956,
+ "loss": 0.8978,
+ "step": 138
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 0.2749930548874636,
+ "learning_rate": 0.00014310968646090883,
+ "loss": 0.924,
+ "step": 139
+ },
+ {
+ "epoch": 1.49,
+ "grad_norm": 0.3097189537380989,
+ "learning_rate": 0.0001423271790405828,
+ "loss": 0.9574,
+ "step": 140
+ },
+ {
+ "epoch": 1.51,
+ "grad_norm": 0.2449218990319832,
+ "learning_rate": 0.00014154150130018866,
+ "loss": 0.8475,
+ "step": 141
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 0.24856388098419674,
+ "learning_rate": 0.0001407527120872557,
+ "loss": 0.9381,
+ "step": 142
+ },
+ {
+ "epoch": 1.53,
+ "grad_norm": 0.3169861882853132,
+ "learning_rate": 0.00013996087048236358,
+ "loss": 0.9141,
+ "step": 143
+ },
+ {
+ "epoch": 1.54,
+ "grad_norm": 0.30689184261103974,
+ "learning_rate": 0.00013916603579471705,
+ "loss": 0.9588,
+ "step": 144
+ },
+ {
+ "epoch": 1.54,
+ "eval_loss": 1.1242448091506958,
+ "eval_runtime": 119.0725,
+ "eval_samples_per_second": 8.39,
+ "eval_steps_per_second": 0.529,
+ "step": 144
+ },
+ {
+ "epoch": 1.55,
+ "grad_norm": 0.2961514212977567,
+ "learning_rate": 0.00013836826755770384,
+ "loss": 0.9371,
+ "step": 145
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 0.30790856503439346,
+ "learning_rate": 0.00013756762552443553,
+ "loss": 0.9612,
+ "step": 146
+ },
+ {
+ "epoch": 1.57,
+ "grad_norm": 0.3517398492864053,
+ "learning_rate": 0.000136764169663272,
+ "loss": 0.9253,
+ "step": 147
+ },
+ {
+ "epoch": 1.58,
+ "grad_norm": 0.26375798832515857,
+ "learning_rate": 0.00013595796015332984,
+ "loss": 0.8977,
+ "step": 148
+ },
+ {
+ "epoch": 1.59,
+ "grad_norm": 0.274348892672977,
+ "learning_rate": 0.00013514905737997473,
+ "loss": 0.8817,
+ "step": 149
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.35917564750751624,
+ "learning_rate": 0.00013433752193029886,
+ "loss": 0.886,
+ "step": 150
+ },
+ {
+ "epoch": 1.61,
+ "grad_norm": 0.38175124377914293,
+ "learning_rate": 0.00013352341458858265,
+ "loss": 0.8576,
+ "step": 151
+ },
+ {
+ "epoch": 1.62,
+ "grad_norm": 0.249633953215678,
+ "learning_rate": 0.00013270679633174218,
+ "loss": 1.0066,
+ "step": 152
+ },
+ {
+ "epoch": 1.63,
+ "grad_norm": 0.33494494430574784,
+ "learning_rate": 0.00013188772832476188,
+ "loss": 0.884,
+ "step": 153
+ },
+ {
+ "epoch": 1.64,
+ "grad_norm": 0.4176467296744032,
+ "learning_rate": 0.00013106627191611332,
+ "loss": 0.9041,
+ "step": 154
+ },
+ {
+ "epoch": 1.65,
+ "grad_norm": 0.27051479454532207,
+ "learning_rate": 0.00013024248863316012,
+ "loss": 0.8764,
+ "step": 155
+ },
+ {
+ "epoch": 1.67,
+ "grad_norm": 0.29302599029848847,
+ "learning_rate": 0.00012941644017754964,
+ "loss": 0.9786,
+ "step": 156
+ },
+ {
+ "epoch": 1.68,
+ "grad_norm": 0.3127378512248151,
+ "learning_rate": 0.00012858818842059145,
+ "loss": 0.9176,
+ "step": 157
+ },
+ {
+ "epoch": 1.69,
+ "grad_norm": 0.40647077063662906,
+ "learning_rate": 0.00012775779539862304,
+ "loss": 0.9387,
+ "step": 158
+ },
+ {
+ "epoch": 1.7,
+ "grad_norm": 0.29290601694481777,
+ "learning_rate": 0.00012692532330836346,
+ "loss": 0.9192,
+ "step": 159
+ },
+ {
+ "epoch": 1.71,
+ "grad_norm": 0.2819168741245354,
+ "learning_rate": 0.0001260908345022547,
+ "loss": 0.9253,
+ "step": 160
+ },
+ {
+ "epoch": 1.72,
+ "grad_norm": 0.3772714091394927,
+ "learning_rate": 0.00012525439148379128,
+ "loss": 0.9264,
+ "step": 161
+ },
+ {
+ "epoch": 1.73,
+ "grad_norm": 0.29399851067321503,
+ "learning_rate": 0.00012441605690283915,
+ "loss": 0.9357,
+ "step": 162
+ },
+ {
+ "epoch": 1.74,
+ "grad_norm": 0.2623180246832513,
+ "learning_rate": 0.00012357589355094275,
+ "loss": 0.8516,
+ "step": 163
+ },
+ {
+ "epoch": 1.75,
+ "grad_norm": 0.27796942024085824,
+ "learning_rate": 0.00012273396435662212,
+ "loss": 0.9328,
+ "step": 164
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 0.3107670297529076,
+ "learning_rate": 0.0001218903323806595,
+ "loss": 0.8769,
+ "step": 165
+ },
+ {
+ "epoch": 1.77,
+ "grad_norm": 0.2865573350738354,
+ "learning_rate": 0.00012104506081137608,
+ "loss": 0.9015,
+ "step": 166
+ },
+ {
+ "epoch": 1.78,
+ "grad_norm": 0.30595087117636693,
+ "learning_rate": 0.00012019821295989912,
+ "loss": 0.94,
+ "step": 167
+ },
+ {
+ "epoch": 1.79,
+ "grad_norm": 0.32540365653257874,
+ "learning_rate": 0.00011934985225541998,
+ "loss": 0.8553,
+ "step": 168
+ },
+ {
+ "epoch": 1.79,
+ "eval_loss": 1.1259374618530273,
+ "eval_runtime": 119.4351,
+ "eval_samples_per_second": 8.364,
+ "eval_steps_per_second": 0.527,
+ "step": 168
+ },
+ {
+ "epoch": 1.8,
+ "grad_norm": 0.3058868303314457,
+ "learning_rate": 0.00011850004224044315,
+ "loss": 0.9074,
+ "step": 169
+ },
+ {
+ "epoch": 1.81,
+ "grad_norm": 0.33266760488242775,
+ "learning_rate": 0.0001176488465660271,
+ "loss": 0.8799,
+ "step": 170
+ },
+ {
+ "epoch": 1.83,
+ "grad_norm": 0.3101183375673487,
+ "learning_rate": 0.00011679632898701649,
+ "loss": 0.9004,
+ "step": 171
+ },
+ {
+ "epoch": 1.84,
+ "grad_norm": 0.31535579418195775,
+ "learning_rate": 0.00011594255335726724,
+ "loss": 0.9238,
+ "step": 172
+ },
+ {
+ "epoch": 1.85,
+ "grad_norm": 0.28341827112854334,
+ "learning_rate": 0.00011508758362486358,
+ "loss": 0.9138,
+ "step": 173
+ },
+ {
+ "epoch": 1.86,
+ "grad_norm": 0.25699888796695625,
+ "learning_rate": 0.00011423148382732853,
+ "loss": 0.9175,
+ "step": 174
+ },
+ {
+ "epoch": 1.87,
+ "grad_norm": 0.29504332662698246,
+ "learning_rate": 0.0001133743180868273,
+ "loss": 0.9023,
+ "step": 175
+ },
+ {
+ "epoch": 1.88,
+ "grad_norm": 0.2993175263873948,
+ "learning_rate": 0.0001125161506053646,
+ "loss": 0.8893,
+ "step": 176
+ },
+ {
+ "epoch": 1.89,
+ "grad_norm": 0.2762659379409218,
+ "learning_rate": 0.00011165704565997593,
+ "loss": 0.9071,
+ "step": 177
+ },
+ {
+ "epoch": 1.9,
+ "grad_norm": 0.23620994229530515,
+ "learning_rate": 0.00011079706759791311,
+ "loss": 0.8796,
+ "step": 178
+ },
+ {
+ "epoch": 1.91,
+ "grad_norm": 0.28317619721877,
+ "learning_rate": 0.00010993628083182467,
+ "loss": 0.8983,
+ "step": 179
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 0.3252854551640304,
+ "learning_rate": 0.00010907474983493144,
+ "loss": 0.8947,
+ "step": 180
+ },
+ {
+ "epoch": 1.93,
+ "grad_norm": 0.2579136274422669,
+ "learning_rate": 0.00010821253913619726,
+ "loss": 0.8726,
+ "step": 181
+ },
+ {
+ "epoch": 1.94,
+ "grad_norm": 0.27201912720918364,
+ "learning_rate": 0.00010734971331549603,
+ "loss": 0.891,
+ "step": 182
+ },
+ {
+ "epoch": 1.95,
+ "grad_norm": 0.41257277193589503,
+ "learning_rate": 0.0001064863369987743,
+ "loss": 0.9188,
+ "step": 183
+ },
+ {
+ "epoch": 1.96,
+ "grad_norm": 0.264920112831242,
+ "learning_rate": 0.00010562247485321115,
+ "loss": 0.8761,
+ "step": 184
+ },
+ {
+ "epoch": 1.98,
+ "grad_norm": 0.28166441056422037,
+ "learning_rate": 0.00010475819158237425,
+ "loss": 0.8805,
+ "step": 185
+ },
+ {
+ "epoch": 1.99,
+ "grad_norm": 0.2818961139392159,
+ "learning_rate": 0.00010389355192137377,
+ "loss": 0.8934,
+ "step": 186
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 0.27424787600345923,
+ "learning_rate": 0.00010302862063201367,
+ "loss": 0.9237,
+ "step": 187
+ },
+ {
+ "epoch": 2.01,
+ "grad_norm": 0.25570082666079225,
+ "learning_rate": 0.00010216346249794087,
+ "loss": 0.8656,
+ "step": 188
+ },
+ {
+ "epoch": 2.02,
+ "grad_norm": 0.2712359904481713,
+ "learning_rate": 0.0001012981423197931,
+ "loss": 0.7627,
+ "step": 189
+ },
+ {
+ "epoch": 2.03,
+ "grad_norm": 0.25054404547068676,
+ "learning_rate": 0.00010043272491034523,
+ "loss": 0.8142,
+ "step": 190
+ },
+ {
+ "epoch": 2.04,
+ "grad_norm": 0.28520868420260026,
+ "learning_rate": 9.956727508965481e-05,
+ "loss": 0.7953,
+ "step": 191
+ },
+ {
+ "epoch": 2.05,
+ "grad_norm": 0.29413880984694873,
+ "learning_rate": 9.870185768020693e-05,
+ "loss": 0.8231,
+ "step": 192
+ },
+ {
+ "epoch": 2.05,
+ "eval_loss": 1.144862413406372,
+ "eval_runtime": 119.3004,
+ "eval_samples_per_second": 8.374,
+ "eval_steps_per_second": 0.528,
+ "step": 192
+ },
+ {
+ "epoch": 2.06,
+ "grad_norm": 0.28378300985247035,
+ "learning_rate": 9.783653750205915e-05,
+ "loss": 0.7478,
+ "step": 193
+ },
+ {
+ "epoch": 2.07,
+ "grad_norm": 0.31792721348179676,
+ "learning_rate": 9.697137936798634e-05,
+ "loss": 0.7961,
+ "step": 194
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 0.3291666436295964,
+ "learning_rate": 9.610644807862625e-05,
+ "loss": 0.7434,
+ "step": 195
+ },
+ {
+ "epoch": 2.09,
+ "grad_norm": 0.301579259001567,
+ "learning_rate": 9.524180841762577e-05,
+ "loss": 0.7779,
+ "step": 196
+ },
+ {
+ "epoch": 2.1,
+ "grad_norm": 0.30252161240414444,
+ "learning_rate": 9.437752514678887e-05,
+ "loss": 0.7689,
+ "step": 197
+ },
+ {
+ "epoch": 2.11,
+ "grad_norm": 0.3350657085129171,
+ "learning_rate": 9.35136630012257e-05,
+ "loss": 0.7574,
+ "step": 198
+ },
+ {
+ "epoch": 2.12,
+ "grad_norm": 0.3053109929956358,
+ "learning_rate": 9.265028668450402e-05,
+ "loss": 0.7729,
+ "step": 199
+ },
+ {
+ "epoch": 2.14,
+ "grad_norm": 0.30367223609567207,
+ "learning_rate": 9.178746086380275e-05,
+ "loss": 0.8111,
+ "step": 200
+ },
+ {
+ "epoch": 2.15,
+ "grad_norm": 0.3366440949136126,
+ "learning_rate": 9.092525016506858e-05,
+ "loss": 0.7986,
+ "step": 201
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 0.3228036608413652,
+ "learning_rate": 9.006371916817534e-05,
+ "loss": 0.8382,
+ "step": 202
+ },
+ {
+ "epoch": 2.17,
+ "grad_norm": 0.2919040789403488,
+ "learning_rate": 8.920293240208694e-05,
+ "loss": 0.7696,
+ "step": 203
+ },
+ {
+ "epoch": 2.18,
+ "grad_norm": 0.30084198177583166,
+ "learning_rate": 8.83429543400241e-05,
+ "loss": 0.7671,
+ "step": 204
+ },
+ {
+ "epoch": 2.19,
+ "grad_norm": 0.33931609000743107,
+ "learning_rate": 8.748384939463543e-05,
+ "loss": 0.7553,
+ "step": 205
+ },
+ {
+ "epoch": 2.2,
+ "grad_norm": 0.30413284924824485,
+ "learning_rate": 8.662568191317273e-05,
+ "loss": 0.7324,
+ "step": 206
+ },
+ {
+ "epoch": 2.21,
+ "grad_norm": 0.3014038998090481,
+ "learning_rate": 8.57685161726715e-05,
+ "loss": 0.7567,
+ "step": 207
+ },
+ {
+ "epoch": 2.22,
+ "grad_norm": 0.3176466329519527,
+ "learning_rate": 8.491241637513644e-05,
+ "loss": 0.8222,
+ "step": 208
+ },
+ {
+ "epoch": 2.23,
+ "grad_norm": 0.29981213041628285,
+ "learning_rate": 8.405744664273278e-05,
+ "loss": 0.7077,
+ "step": 209
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 0.2937916452228122,
+ "learning_rate": 8.320367101298351e-05,
+ "loss": 0.7231,
+ "step": 210
+ },
+ {
+ "epoch": 2.25,
+ "grad_norm": 0.32040684171320816,
+ "learning_rate": 8.235115343397295e-05,
+ "loss": 0.7556,
+ "step": 211
+ },
+ {
+ "epoch": 2.26,
+ "grad_norm": 0.31083028085316033,
+ "learning_rate": 8.149995775955686e-05,
+ "loss": 0.7514,
+ "step": 212
+ },
+ {
+ "epoch": 2.27,
+ "grad_norm": 0.3215465383581194,
+ "learning_rate": 8.065014774458003e-05,
+ "loss": 0.7933,
+ "step": 213
+ },
+ {
+ "epoch": 2.28,
+ "grad_norm": 0.3081200259196015,
+ "learning_rate": 7.980178704010089e-05,
+ "loss": 0.8062,
+ "step": 214
+ },
+ {
+ "epoch": 2.3,
+ "grad_norm": 0.3333248296288759,
+ "learning_rate": 7.895493918862396e-05,
+ "loss": 0.7784,
+ "step": 215
+ },
+ {
+ "epoch": 2.31,
+ "grad_norm": 0.3301326097292383,
+ "learning_rate": 7.810966761934053e-05,
+ "loss": 0.8154,
+ "step": 216
+ },
+ {
+ "epoch": 2.31,
+ "eval_loss": 1.1513652801513672,
+ "eval_runtime": 119.4371,
+ "eval_samples_per_second": 8.364,
+ "eval_steps_per_second": 0.527,
+ "step": 216
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 0.3166760836422428,
+ "learning_rate": 7.726603564337791e-05,
+ "loss": 0.7486,
+ "step": 217
+ },
+ {
+ "epoch": 2.33,
+ "grad_norm": 0.31309757318131876,
+ "learning_rate": 7.642410644905726e-05,
+ "loss": 0.771,
+ "step": 218
+ },
+ {
+ "epoch": 2.34,
+ "grad_norm": 0.36968796131043985,
+ "learning_rate": 7.558394309716088e-05,
+ "loss": 0.8051,
+ "step": 219
+ },
+ {
+ "epoch": 2.35,
+ "grad_norm": 0.27537675917328025,
+ "learning_rate": 7.474560851620873e-05,
+ "loss": 0.7536,
+ "step": 220
+ },
+ {
+ "epoch": 2.36,
+ "grad_norm": 0.2878011945022053,
+ "learning_rate": 7.390916549774536e-05,
+ "loss": 0.8126,
+ "step": 221
+ },
+ {
+ "epoch": 2.37,
+ "grad_norm": 0.3172405217395398,
+ "learning_rate": 7.307467669163655e-05,
+ "loss": 0.8156,
+ "step": 222
+ },
+ {
+ "epoch": 2.38,
+ "grad_norm": 0.3183651086957915,
+ "learning_rate": 7.224220460137701e-05,
+ "loss": 0.7821,
+ "step": 223
+ },
+ {
+ "epoch": 2.39,
+ "grad_norm": 0.3318078467573977,
+ "learning_rate": 7.141181157940859e-05,
+ "loss": 0.7993,
+ "step": 224
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 0.28446170407344085,
+ "learning_rate": 7.058355982245037e-05,
+ "loss": 0.7987,
+ "step": 225
+ },
+ {
+ "epoch": 2.41,
+ "grad_norm": 0.33568352702219995,
+ "learning_rate": 6.97575113668399e-05,
+ "loss": 0.773,
+ "step": 226
+ },
+ {
+ "epoch": 2.42,
+ "grad_norm": 0.30820575901544944,
+ "learning_rate": 6.893372808388675e-05,
+ "loss": 0.813,
+ "step": 227
+ },
+ {
+ "epoch": 2.43,
+ "grad_norm": 0.3121364386024255,
+ "learning_rate": 6.811227167523815e-05,
+ "loss": 0.7716,
+ "step": 228
+ },
+ {
+ "epoch": 2.44,
+ "grad_norm": 0.3211455560922844,
+ "learning_rate": 6.729320366825784e-05,
+ "loss": 0.7577,
+ "step": 229
+ },
+ {
+ "epoch": 2.46,
+ "grad_norm": 0.3315601260165869,
+ "learning_rate": 6.647658541141735e-05,
+ "loss": 0.779,
+ "step": 230
+ },
+ {
+ "epoch": 2.47,
+ "grad_norm": 0.35482236759964675,
+ "learning_rate": 6.566247806970119e-05,
+ "loss": 0.7936,
+ "step": 231
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 0.3318703205331905,
+ "learning_rate": 6.485094262002529e-05,
+ "loss": 0.7721,
+ "step": 232
+ },
+ {
+ "epoch": 2.49,
+ "grad_norm": 0.313412585518615,
+ "learning_rate": 6.404203984667019e-05,
+ "loss": 0.7333,
+ "step": 233
+ },
+ {
+ "epoch": 2.5,
+ "grad_norm": 0.3389693444254627,
+ "learning_rate": 6.323583033672799e-05,
+ "loss": 0.6991,
+ "step": 234
+ },
+ {
+ "epoch": 2.51,
+ "grad_norm": 0.33056782619334757,
+ "learning_rate": 6.243237447556449e-05,
+ "loss": 0.7872,
+ "step": 235
+ },
+ {
+ "epoch": 2.52,
+ "grad_norm": 0.3064085209522584,
+ "learning_rate": 6.163173244229619e-05,
+ "loss": 0.7713,
+ "step": 236
+ },
+ {
+ "epoch": 2.53,
+ "grad_norm": 0.3109445125421656,
+ "learning_rate": 6.083396420528298e-05,
+ "loss": 0.8228,
+ "step": 237
+ },
+ {
+ "epoch": 2.54,
+ "grad_norm": 0.35767207742703394,
+ "learning_rate": 6.0039129517636435e-05,
+ "loss": 0.8167,
+ "step": 238
+ },
+ {
+ "epoch": 2.55,
+ "grad_norm": 0.32869196909020376,
+ "learning_rate": 5.924728791274432e-05,
+ "loss": 0.7893,
+ "step": 239
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 0.31178216743238674,
+ "learning_rate": 5.845849869981137e-05,
+ "loss": 0.7354,
+ "step": 240
+ },
+ {
+ "epoch": 2.56,
+ "eval_loss": 1.1470853090286255,
+ "eval_runtime": 119.0749,
+ "eval_samples_per_second": 8.39,
+ "eval_steps_per_second": 0.529,
+ "step": 240
+ },
+ {
+ "epoch": 2.57,
+ "grad_norm": 0.3146586486940167,
+ "learning_rate": 5.7672820959417254e-05,
+ "loss": 0.785,
+ "step": 241
+ },
+ {
+ "epoch": 2.58,
+ "grad_norm": 0.3309473634570162,
+ "learning_rate": 5.68903135390912e-05,
+ "loss": 0.7007,
+ "step": 242
+ },
+ {
+ "epoch": 2.59,
+ "grad_norm": 0.2927704203363025,
+ "learning_rate": 5.611103504890444e-05,
+ "loss": 0.778,
+ "step": 243
+ },
+ {
+ "epoch": 2.6,
+ "grad_norm": 0.31346541530480915,
+ "learning_rate": 5.533504385708024e-05,
+ "loss": 0.7272,
+ "step": 244
+ },
+ {
+ "epoch": 2.62,
+ "grad_norm": 0.2996345434845278,
+ "learning_rate": 5.456239808562209e-05,
+ "loss": 0.8091,
+ "step": 245
+ },
+ {
+ "epoch": 2.63,
+ "grad_norm": 0.29407937930772826,
+ "learning_rate": 5.379315560596038e-05,
+ "loss": 0.7666,
+ "step": 246
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.30530254935425627,
+ "learning_rate": 5.3027374034617785e-05,
+ "loss": 0.7982,
+ "step": 247
+ },
+ {
+ "epoch": 2.65,
+ "grad_norm": 0.3298149075133802,
+ "learning_rate": 5.226511072889371e-05,
+ "loss": 0.7962,
+ "step": 248
+ },
+ {
+ "epoch": 2.66,
+ "grad_norm": 0.33155001378615223,
+ "learning_rate": 5.1506422782568345e-05,
+ "loss": 0.8087,
+ "step": 249
+ },
+ {
+ "epoch": 2.67,
+ "grad_norm": 0.32891369446509405,
+ "learning_rate": 5.0751367021626215e-05,
+ "loss": 0.7702,
+ "step": 250
+ },
+ {
+ "epoch": 2.68,
+ "grad_norm": 0.3042328939887202,
+ "learning_rate": 5.000000000000002e-05,
+ "loss": 0.7924,
+ "step": 251
+ },
+ {
+ "epoch": 2.69,
+ "grad_norm": 0.3037799376581133,
+ "learning_rate": 4.9252377995334444e-05,
+ "loss": 0.7852,
+ "step": 252
+ },
+ {
+ "epoch": 2.7,
+ "grad_norm": 0.3435430445603929,
+ "learning_rate": 4.85085570047713e-05,
+ "loss": 0.7501,
+ "step": 253
+ },
+ {
+ "epoch": 2.71,
+ "grad_norm": 0.3072160193979946,
+ "learning_rate": 4.776859274075506e-05,
+ "loss": 0.7462,
+ "step": 254
+ },
+ {
+ "epoch": 2.72,
+ "grad_norm": 0.3223586439500028,
+ "learning_rate": 4.703254062686017e-05,
+ "loss": 0.775,
+ "step": 255
+ },
+ {
+ "epoch": 2.73,
+ "grad_norm": 0.3270406403084203,
+ "learning_rate": 4.630045579363957e-05,
+ "loss": 0.8306,
+ "step": 256
+ },
+ {
+ "epoch": 2.74,
+ "grad_norm": 0.3360192842512657,
+ "learning_rate": 4.557239307449561e-05,
+ "loss": 0.7697,
+ "step": 257
+ },
+ {
+ "epoch": 2.75,
+ "grad_norm": 0.34282816479900324,
+ "learning_rate": 4.484840700157295e-05,
+ "loss": 0.7654,
+ "step": 258
+ },
+ {
+ "epoch": 2.77,
+ "grad_norm": 0.30039142762313786,
+ "learning_rate": 4.412855180167406e-05,
+ "loss": 0.7703,
+ "step": 259
+ },
+ {
+ "epoch": 2.78,
+ "grad_norm": 0.34307884673711425,
+ "learning_rate": 4.3412881392197526e-05,
+ "loss": 0.7993,
+ "step": 260
+ },
+ {
+ "epoch": 2.79,
+ "grad_norm": 0.33685538845268104,
+ "learning_rate": 4.270144937709981e-05,
+ "loss": 0.7866,
+ "step": 261
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 0.33166767859224683,
+ "learning_rate": 4.19943090428802e-05,
+ "loss": 0.8083,
+ "step": 262
+ },
+ {
+ "epoch": 2.81,
+ "grad_norm": 0.3086370003245581,
+ "learning_rate": 4.129151335458957e-05,
+ "loss": 0.7938,
+ "step": 263
+ },
+ {
+ "epoch": 2.82,
+ "grad_norm": 0.3715649674817313,
+ "learning_rate": 4.059311495186338e-05,
+ "loss": 0.7577,
+ "step": 264
+ },
+ {
+ "epoch": 2.82,
+ "eval_loss": 1.1478512287139893,
+ "eval_runtime": 119.1178,
+ "eval_samples_per_second": 8.387,
+ "eval_steps_per_second": 0.529,
+ "step": 264
+ },
+ {
+ "epoch": 2.83,
+ "grad_norm": 0.3298033298390841,
+ "learning_rate": 3.9899166144978904e-05,
+ "loss": 0.8296,
+ "step": 265
+ },
+ {
+ "epoch": 2.84,
+ "grad_norm": 0.3294808666769515,
+ "learning_rate": 3.920971891093718e-05,
+ "loss": 0.8206,
+ "step": 266
+ },
+ {
+ "epoch": 2.85,
+ "grad_norm": 0.3239672501165848,
+ "learning_rate": 3.852482488956992e-05,
+ "loss": 0.8116,
+ "step": 267
+ },
+ {
+ "epoch": 2.86,
+ "grad_norm": 0.3286742994048133,
+ "learning_rate": 3.784453537967161e-05,
+ "loss": 0.8096,
+ "step": 268
+ },
+ {
+ "epoch": 2.87,
+ "grad_norm": 0.31259050250842946,
+ "learning_rate": 3.7168901335157315e-05,
+ "loss": 0.7669,
+ "step": 269
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 0.3308991711135206,
+ "learning_rate": 3.649797336124615e-05,
+ "loss": 0.8041,
+ "step": 270
+ },
+ {
+ "epoch": 2.89,
+ "grad_norm": 0.32757727002633424,
+ "learning_rate": 3.583180171067101e-05,
+ "loss": 0.7673,
+ "step": 271
+ },
+ {
+ "epoch": 2.9,
+ "grad_norm": 0.3342551756453125,
+ "learning_rate": 3.517043627991441e-05,
+ "loss": 0.8005,
+ "step": 272
+ },
+ {
+ "epoch": 2.91,
+ "grad_norm": 0.31643754309861705,
+ "learning_rate": 3.45139266054715e-05,
+ "loss": 0.787,
+ "step": 273
+ },
+ {
+ "epoch": 2.93,
+ "grad_norm": 0.3140452683879005,
+ "learning_rate": 3.3862321860139576e-05,
+ "loss": 0.7888,
+ "step": 274
+ },
+ {
+ "epoch": 2.94,
+ "grad_norm": 0.30706221155036223,
+ "learning_rate": 3.3215670849335155e-05,
+ "loss": 0.827,
+ "step": 275
+ },
+ {
+ "epoch": 2.95,
+ "grad_norm": 0.3185483102727301,
+ "learning_rate": 3.257402200743821e-05,
+ "loss": 0.7779,
+ "step": 276
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 0.3032818796307545,
+ "learning_rate": 3.19374233941647e-05,
+ "loss": 0.7993,
+ "step": 277
+ },
+ {
+ "epoch": 2.97,
+ "grad_norm": 0.3057758504695884,
+ "learning_rate": 3.130592269096671e-05,
+ "loss": 0.768,
+ "step": 278
+ },
+ {
+ "epoch": 2.98,
+ "grad_norm": 0.3245404038219604,
+ "learning_rate": 3.0679567197461134e-05,
+ "loss": 0.7706,
+ "step": 279
+ },
+ {
+ "epoch": 2.99,
+ "grad_norm": 0.3376535123919746,
+ "learning_rate": 3.005840382788685e-05,
+ "loss": 0.7825,
+ "step": 280
+ },
+ {
+ "epoch": 3.0,
+ "grad_norm": 0.34483227716329967,
+ "learning_rate": 2.944247910759097e-05,
+ "loss": 0.7725,
+ "step": 281
+ },
+ {
+ "epoch": 3.01,
+ "grad_norm": 0.30532824560617583,
+ "learning_rate": 2.8831839169543996e-05,
+ "loss": 0.7228,
+ "step": 282
+ },
+ {
+ "epoch": 3.02,
+ "grad_norm": 0.31015055336513103,
+ "learning_rate": 2.8226529750884402e-05,
+ "loss": 0.6507,
+ "step": 283
+ },
+ {
+ "epoch": 3.03,
+ "grad_norm": 0.3125936248719555,
+ "learning_rate": 2.7626596189492983e-05,
+ "loss": 0.734,
+ "step": 284
+ },
+ {
+ "epoch": 3.04,
+ "grad_norm": 0.32131974094099536,
+ "learning_rate": 2.7032083420597e-05,
+ "loss": 0.7168,
+ "step": 285
+ },
+ {
+ "epoch": 3.05,
+ "grad_norm": 0.3188571782464755,
+ "learning_rate": 2.6443035973404496e-05,
+ "loss": 0.6591,
+ "step": 286
+ },
+ {
+ "epoch": 3.06,
+ "grad_norm": 0.33177480330954007,
+ "learning_rate": 2.585949796776912e-05,
+ "loss": 0.6965,
+ "step": 287
+ },
+ {
+ "epoch": 3.07,
+ "grad_norm": 0.38345252271104163,
+ "learning_rate": 2.528151311088537e-05,
+ "loss": 0.6647,
+ "step": 288
+ },
+ {
+ "epoch": 3.07,
+ "eval_loss": 1.1923209428787231,
+ "eval_runtime": 118.9649,
+ "eval_samples_per_second": 8.397,
+ "eval_steps_per_second": 0.53,
+ "step": 288
+ },
+ {
+ "epoch": 3.09,
+ "grad_norm": 0.3519739075006545,
+ "learning_rate": 2.4709124694015116e-05,
+ "loss": 0.7169,
+ "step": 289
+ },
+ {
+ "epoch": 3.1,
+ "grad_norm": 0.34731968650913336,
+ "learning_rate": 2.4142375589244957e-05,
+ "loss": 0.7161,
+ "step": 290
+ },
+ {
+ "epoch": 3.11,
+ "grad_norm": 0.33778912337210615,
+ "learning_rate": 2.3581308246275103e-05,
+ "loss": 0.7155,
+ "step": 291
+ },
+ {
+ "epoch": 3.12,
+ "grad_norm": 0.3209719266692497,
+ "learning_rate": 2.302596468923981e-05,
+ "loss": 0.672,
+ "step": 292
+ },
+ {
+ "epoch": 3.13,
+ "grad_norm": 0.3450647699674907,
+ "learning_rate": 2.247638651355991e-05,
+ "loss": 0.696,
+ "step": 293
+ },
+ {
+ "epoch": 3.14,
+ "grad_norm": 0.34077271836911865,
+ "learning_rate": 2.1932614882827197e-05,
+ "loss": 0.6947,
+ "step": 294
+ },
+ {
+ "epoch": 3.15,
+ "grad_norm": 0.35952846733253163,
+ "learning_rate": 2.139469052572127e-05,
+ "loss": 0.6934,
+ "step": 295
+ },
+ {
+ "epoch": 3.16,
+ "grad_norm": 0.32425812401737625,
+ "learning_rate": 2.0862653732958915e-05,
+ "loss": 0.6803,
+ "step": 296
+ },
+ {
+ "epoch": 3.17,
+ "grad_norm": 0.3354776806257836,
+ "learning_rate": 2.03365443542764e-05,
+ "loss": 0.6847,
+ "step": 297
+ },
+ {
+ "epoch": 3.18,
+ "grad_norm": 0.35880539160768654,
+ "learning_rate": 1.981640179544466e-05,
+ "loss": 0.6447,
+ "step": 298
+ },
+ {
+ "epoch": 3.19,
+ "grad_norm": 0.3413724716725652,
+ "learning_rate": 1.93022650153178e-05,
+ "loss": 0.6224,
+ "step": 299
+ },
+ {
+ "epoch": 3.2,
+ "grad_norm": 0.3424458691012398,
+ "learning_rate": 1.879417252291502e-05,
+ "loss": 0.6699,
+ "step": 300
+ },
+ {
+ "epoch": 3.21,
+ "grad_norm": 0.3382667696983362,
+ "learning_rate": 1.829216237453637e-05,
+ "loss": 0.6797,
+ "step": 301
+ },
+ {
+ "epoch": 3.22,
+ "grad_norm": 0.324989093083984,
+ "learning_rate": 1.7796272170912253e-05,
+ "loss": 0.691,
+ "step": 302
+ },
+ {
+ "epoch": 3.23,
+ "grad_norm": 0.32204008590962435,
+ "learning_rate": 1.730653905438714e-05,
+ "loss": 0.6636,
+ "step": 303
+ },
+ {
+ "epoch": 3.25,
+ "grad_norm": 0.34081693500058535,
+ "learning_rate": 1.6822999706137567e-05,
+ "loss": 0.7285,
+ "step": 304
+ },
+ {
+ "epoch": 3.26,
+ "grad_norm": 0.3745986061815685,
+ "learning_rate": 1.634569034342476e-05,
+ "loss": 0.6896,
+ "step": 305
+ },
+ {
+ "epoch": 3.27,
+ "grad_norm": 0.3460331973628518,
+ "learning_rate": 1.587464671688187e-05,
+ "loss": 0.6988,
+ "step": 306
+ },
+ {
+ "epoch": 3.28,
+ "grad_norm": 0.3548738091123995,
+ "learning_rate": 1.5409904107836358e-05,
+ "loss": 0.7166,
+ "step": 307
+ },
+ {
+ "epoch": 3.29,
+ "grad_norm": 0.3616387758766049,
+ "learning_rate": 1.495149732566723e-05,
+ "loss": 0.7033,
+ "step": 308
+ },
+ {
+ "epoch": 3.3,
+ "grad_norm": 0.3358013853175185,
+ "learning_rate": 1.4499460705197998e-05,
+ "loss": 0.7096,
+ "step": 309
+ },
+ {
+ "epoch": 3.31,
+ "grad_norm": 0.34774337705479985,
+ "learning_rate": 1.4053828104124867e-05,
+ "loss": 0.6695,
+ "step": 310
+ },
+ {
+ "epoch": 3.32,
+ "grad_norm": 0.3273729493403579,
+ "learning_rate": 1.361463290048085e-05,
+ "loss": 0.6893,
+ "step": 311
+ },
+ {
+ "epoch": 3.33,
+ "grad_norm": 0.343572039488577,
+ "learning_rate": 1.3181907990135622e-05,
+ "loss": 0.6928,
+ "step": 312
+ },
+ {
+ "epoch": 3.33,
+ "eval_loss": 1.185637354850769,
+ "eval_runtime": 119.6535,
+ "eval_samples_per_second": 8.349,
+ "eval_steps_per_second": 0.527,
+ "step": 312
+ },
+ {
+ "epoch": 3.34,
+ "grad_norm": 0.36314106359094633,
+ "learning_rate": 1.2755685784331783e-05,
+ "loss": 0.6887,
+ "step": 313
+ },
+ {
+ "epoch": 3.35,
+ "grad_norm": 0.3178995942964272,
+ "learning_rate": 1.2335998207257137e-05,
+ "loss": 0.7163,
+ "step": 314
+ },
+ {
+ "epoch": 3.36,
+ "grad_norm": 0.3530883103274161,
+ "learning_rate": 1.1922876693653585e-05,
+ "loss": 0.6264,
+ "step": 315
+ },
+ {
+ "epoch": 3.37,
+ "grad_norm": 0.34700396777915954,
+ "learning_rate": 1.1516352186462586e-05,
+ "loss": 0.6414,
+ "step": 316
+ },
+ {
+ "epoch": 3.38,
+ "grad_norm": 0.3259373809872755,
+ "learning_rate": 1.1116455134507664e-05,
+ "loss": 0.6461,
+ "step": 317
+ },
+ {
+ "epoch": 3.4,
+ "grad_norm": 0.35544661568277985,
+ "learning_rate": 1.0723215490213634e-05,
+ "loss": 0.6723,
+ "step": 318
+ },
+ {
+ "epoch": 3.41,
+ "grad_norm": 0.3488692435606677,
+ "learning_rate": 1.0336662707363287e-05,
+ "loss": 0.7782,
+ "step": 319
+ },
+ {
+ "epoch": 3.42,
+ "grad_norm": 0.35693983497851145,
+ "learning_rate": 9.95682573889114e-06,
+ "loss": 0.6982,
+ "step": 320
+ },
+ {
+ "epoch": 3.43,
+ "grad_norm": 0.33248426163504435,
+ "learning_rate": 9.583733034714981e-06,
+ "loss": 0.6326,
+ "step": 321
+ },
+ {
+ "epoch": 3.44,
+ "grad_norm": 0.38375415594727097,
+ "learning_rate": 9.217412539604942e-06,
+ "loss": 0.6971,
+ "step": 322
+ },
+ {
+ "epoch": 3.45,
+ "grad_norm": 0.37036824832855825,
+ "learning_rate": 8.857891691090337e-06,
+ "loss": 0.698,
+ "step": 323
+ },
+ {
+ "epoch": 3.46,
+ "grad_norm": 0.3417675223689608,
+ "learning_rate": 8.505197417404687e-06,
+ "loss": 0.6836,
+ "step": 324
+ },
+ {
+ "epoch": 3.47,
+ "grad_norm": 0.3700366171554004,
+ "learning_rate": 8.15935613546872e-06,
+ "loss": 0.6465,
+ "step": 325
+ },
+ {
+ "epoch": 3.48,
+ "grad_norm": 0.3492600384568647,
+ "learning_rate": 7.820393748911791e-06,
+ "loss": 0.6808,
+ "step": 326
+ },
+ {
+ "epoch": 3.49,
+ "grad_norm": 0.36976467848962297,
+ "learning_rate": 7.488335646131628e-06,
+ "loss": 0.6615,
+ "step": 327
+ },
+ {
+ "epoch": 3.5,
+ "grad_norm": 0.3290900616093639,
+ "learning_rate": 7.163206698392744e-06,
+ "loss": 0.7275,
+ "step": 328
+ },
+ {
+ "epoch": 3.51,
+ "grad_norm": 0.3566655771775109,
+ "learning_rate": 6.845031257963619e-06,
+ "loss": 0.7116,
+ "step": 329
+ },
+ {
+ "epoch": 3.52,
+ "grad_norm": 0.320805379756211,
+ "learning_rate": 6.533833156292679e-06,
+ "loss": 0.6879,
+ "step": 330
+ },
+ {
+ "epoch": 3.53,
+ "grad_norm": 0.34337800662165496,
+ "learning_rate": 6.229635702223324e-06,
+ "loss": 0.7163,
+ "step": 331
+ },
+ {
+ "epoch": 3.54,
+ "grad_norm": 0.36367722946797926,
+ "learning_rate": 5.932461680248014e-06,
+ "loss": 0.7021,
+ "step": 332
+ },
+ {
+ "epoch": 3.56,
+ "grad_norm": 0.36522399153203633,
+ "learning_rate": 5.6423333488018095e-06,
+ "loss": 0.6861,
+ "step": 333
+ },
+ {
+ "epoch": 3.57,
+ "grad_norm": 0.365852241324378,
+ "learning_rate": 5.359272438595153e-06,
+ "loss": 0.6885,
+ "step": 334
+ },
+ {
+ "epoch": 3.58,
+ "grad_norm": 0.3614672831032901,
+ "learning_rate": 5.083300150986259e-06,
+ "loss": 0.6783,
+ "step": 335
+ },
+ {
+ "epoch": 3.59,
+ "grad_norm": 0.3642169019209535,
+ "learning_rate": 4.8144371563930476e-06,
+ "loss": 0.731,
+ "step": 336
+ },
+ {
+ "epoch": 3.59,
+ "eval_loss": 1.1889785528182983,
+ "eval_runtime": 119.1203,
+ "eval_samples_per_second": 8.386,
+ "eval_steps_per_second": 0.529,
+ "step": 336
+ },
+ {
+ "epoch": 3.6,
+ "grad_norm": 0.34161446199658746,
+ "learning_rate": 4.552703592745033e-06,
+ "loss": 0.7138,
+ "step": 337
+ },
+ {
+ "epoch": 3.61,
+ "grad_norm": 0.35734876291405965,
+ "learning_rate": 4.298119063974914e-06,
+ "loss": 0.6725,
+ "step": 338
+ },
+ {
+ "epoch": 3.62,
+ "grad_norm": 0.32914706405183236,
+ "learning_rate": 4.050702638550275e-06,
+ "loss": 0.6961,
+ "step": 339
+ },
+ {
+ "epoch": 3.63,
+ "grad_norm": 0.3570873592035516,
+ "learning_rate": 3.810472848045266e-06,
+ "loss": 0.6514,
+ "step": 340
+ },
+ {
+ "epoch": 3.64,
+ "grad_norm": 0.3597890086242411,
+ "learning_rate": 3.5774476857527107e-06,
+ "loss": 0.6962,
+ "step": 341
+ },
+ {
+ "epoch": 3.65,
+ "grad_norm": 0.38480539648857937,
+ "learning_rate": 3.3516446053363015e-06,
+ "loss": 0.6686,
+ "step": 342
+ },
+ {
+ "epoch": 3.66,
+ "grad_norm": 0.3525111961446683,
+ "learning_rate": 3.133080519523368e-06,
+ "loss": 0.6667,
+ "step": 343
+ },
+ {
+ "epoch": 3.67,
+ "grad_norm": 0.3735012861865468,
+ "learning_rate": 2.921771798838069e-06,
+ "loss": 0.7137,
+ "step": 344
+ },
+ {
+ "epoch": 3.68,
+ "grad_norm": 0.35587547424949395,
+ "learning_rate": 2.717734270375272e-06,
+ "loss": 0.6743,
+ "step": 345
+ },
+ {
+ "epoch": 3.69,
+ "grad_norm": 0.3657817512714091,
+ "learning_rate": 2.520983216615047e-06,
+ "loss": 0.6373,
+ "step": 346
+ },
+ {
+ "epoch": 3.7,
+ "grad_norm": 0.3509346866409975,
+ "learning_rate": 2.3315333742780942e-06,
+ "loss": 0.6703,
+ "step": 347
+ },
+ {
+ "epoch": 3.72,
+ "grad_norm": 0.3459857927296872,
+ "learning_rate": 2.1493989332218468e-06,
+ "loss": 0.7223,
+ "step": 348
+ },
+ {
+ "epoch": 3.73,
+ "grad_norm": 0.38016379979574644,
+ "learning_rate": 1.974593535377722e-06,
+ "loss": 0.7024,
+ "step": 349
+ },
+ {
+ "epoch": 3.74,
+ "grad_norm": 0.3191443635681887,
+ "learning_rate": 1.8071302737293295e-06,
+ "loss": 0.6933,
+ "step": 350
+ },
+ {
+ "epoch": 3.75,
+ "grad_norm": 0.3586330010035836,
+ "learning_rate": 1.6470216913317626e-06,
+ "loss": 0.7001,
+ "step": 351
+ },
+ {
+ "epoch": 3.76,
+ "grad_norm": 0.3850836045039222,
+ "learning_rate": 1.4942797803721543e-06,
+ "loss": 0.61,
+ "step": 352
+ },
+ {
+ "epoch": 3.77,
+ "grad_norm": 0.34888020988423163,
+ "learning_rate": 1.348915981271437e-06,
+ "loss": 0.6528,
+ "step": 353
+ },
+ {
+ "epoch": 3.78,
+ "grad_norm": 0.354531551308838,
+ "learning_rate": 1.2109411818274852e-06,
+ "loss": 0.7128,
+ "step": 354
+ },
+ {
+ "epoch": 3.79,
+ "grad_norm": 0.38330787603530414,
+ "learning_rate": 1.0803657163995895e-06,
+ "loss": 0.643,
+ "step": 355
+ },
+ {
+ "epoch": 3.8,
+ "grad_norm": 0.3432828481582299,
+ "learning_rate": 9.57199365134387e-07,
+ "loss": 0.6318,
+ "step": 356
+ },
+ {
+ "epoch": 3.81,
+ "grad_norm": 0.37153831756155836,
+ "learning_rate": 8.41451353233369e-07,
+ "loss": 0.694,
+ "step": 357
+ },
+ {
+ "epoch": 3.82,
+ "grad_norm": 0.3701195226107489,
+ "learning_rate": 7.331303502618903e-07,
+ "loss": 0.6561,
+ "step": 358
+ },
+ {
+ "epoch": 3.83,
+ "grad_norm": 0.3831847969050675,
+ "learning_rate": 6.322444694998319e-07,
+ "loss": 0.7167,
+ "step": 359
+ },
+ {
+ "epoch": 3.84,
+ "grad_norm": 0.3382934858827101,
+ "learning_rate": 5.388012673338661e-07,
+ "loss": 0.7193,
+ "step": 360
+ },
+ {
+ "epoch": 3.84,
+ "eval_loss": 1.191129207611084,
+ "eval_runtime": 119.5001,
+ "eval_samples_per_second": 8.36,
+ "eval_steps_per_second": 0.527,
+ "step": 360
+ },
+ {
+ "epoch": 3.85,
+ "grad_norm": 0.35121545616673777,
+ "learning_rate": 4.5280774269154115e-07,
+ "loss": 0.6838,
+ "step": 361
+ },
+ {
+ "epoch": 3.86,
+ "grad_norm": 0.3677398099025592,
+ "learning_rate": 3.742703365170241e-07,
+ "loss": 0.6364,
+ "step": 362
+ },
+ {
+ "epoch": 3.88,
+ "grad_norm": 0.36901780764712055,
+ "learning_rate": 3.0319493128866396e-07,
+ "loss": 0.6827,
+ "step": 363
+ },
+ {
+ "epoch": 3.89,
+ "grad_norm": 0.35056873374955655,
+ "learning_rate": 2.395868505784438e-07,
+ "loss": 0.6432,
+ "step": 364
+ },
+ {
+ "epoch": 3.9,
+ "grad_norm": 0.3627218430880944,
+ "learning_rate": 1.83450858653178e-07,
+ "loss": 0.7243,
+ "step": 365
+ },
+ {
+ "epoch": 3.91,
+ "grad_norm": 0.32428680992451525,
+ "learning_rate": 1.3479116011769767e-07,
+ "loss": 0.6574,
+ "step": 366
+ },
+ {
+ "epoch": 3.92,
+ "grad_norm": 0.349287012328286,
+ "learning_rate": 9.361139959993549e-08,
+ "loss": 0.7048,
+ "step": 367
+ },
+ {
+ "epoch": 3.93,
+ "grad_norm": 0.3701389678746645,
+ "learning_rate": 5.991466147791113e-08,
+ "loss": 0.6742,
+ "step": 368
+ },
+ {
+ "epoch": 3.94,
+ "grad_norm": 0.3562569316390435,
+ "learning_rate": 3.370346964876036e-08,
+ "loss": 0.6747,
+ "step": 369
+ },
+ {
+ "epoch": 3.95,
+ "grad_norm": 0.35498414175449955,
+ "learning_rate": 1.4979787339619578e-08,
+ "loss": 0.6466,
+ "step": 370
+ },
+ {
+ "epoch": 3.96,
+ "grad_norm": 0.353867118979457,
+ "learning_rate": 3.745016960665648e-09,
+ "loss": 0.6586,
+ "step": 371
+ },
+ {
+ "epoch": 3.97,
+ "grad_norm": 0.3411723289198462,
+ "learning_rate": 0.0,
+ "loss": 0.6657,
+ "step": 372
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 372,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 4,
+ "save_steps": 93,
+ "total_flos": 6.890718948426252e+18,
+ "train_batch_size": 4,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-372/training_args.bin b/checkpoint-372/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..bc11601d02e30533659dbf12ba92faed1ba25b5a
--- /dev/null
+++ b/checkpoint-372/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c56515a18cd914d4eee44c09952d3a756ea623b0b6e69e8dfaeb0dbc7b665f46
+size 6776
diff --git a/checkpoint-372/zero_to_fp32.py b/checkpoint-372/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..49b846633d6eb1e836e34681e44033581f4edb7b
--- /dev/null
+++ b/checkpoint-372/zero_to_fp32.py
@@ -0,0 +1,592 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+ total_files = len(files)
+ state_dicts = []
+ for f in files:
+ state_dict = torch.load(f, map_location=device)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ if zero_stage <= 2:
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ elif zero_stage == 3:
+ # if there is more than one param group, there will be multiple flattened tensors - one
+ # flattened tensor per group - for simplicity merge them into a single tensor
+ #
+ # XXX: could make the script more memory efficient for when there are multiple groups - it
+ # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+ fp32_flat_groups = [
+ torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+ ]
+
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ model_files = get_model_state_files(ds_checkpoint_dir)
+
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ for name, shape in param_shapes.items():
+
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # XXX: memory usage doubles here
+ state_dict[name] = torch.cat(
+ tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+ 0).narrow(0, 0, unpartitioned_numel).view(shape)
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+ you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ """
+
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+ print(f"Saving fp32 state dict to {output_file}")
+ torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info(f"Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info(f"Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument(
+ "output_file",
+ type=str,
+ help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)
diff --git a/checkpoint-448/README.md b/checkpoint-448/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..cf214dc8241cd3fa9db952d247cbfcba94fa04e6
--- /dev/null
+++ b/checkpoint-448/README.md
@@ -0,0 +1,1091 @@
+---
+library_name: peft
+base_model: google/gemma-2b
+---
+
+# Model Card for Model ID
+
+Fine-tuned on python
+
+
+## Model Details
+
+### Model Description
+
+
+### Model Sources [optional]
+Gemma-2b trained on python-oasst dataset
+
+## Uses
+
+
+## Training Details
+
+### Training Data
+
+{
+ "_timestamp": 1711018613.433522,
+ "train/grad_norm": 0.1240904619429259,
+ "train/global_step": 232,
+ "eval/steps_per_second": 2.894,
+ "_step": 232,
+ "_runtime": 2545.4226660728455,
+ "eval/loss": 1.189491629600525,
+ "eval/runtime": 1805.8574,
+ "train/learning_rate": 0.000014800637958532697,
+ "eval/samples_per_second": 23.152,
+ "_wandb.runtime": 2547,
+ "train/loss": 1.0436,
+ "train/epoch": 0.01
+}
+
+### Results
+
+
+#### Summary
+
+
+
+
+## Technical Specifications [optional]
+{
+ "bf16": {
+ "desc": null,
+ "value": true
+ },
+ "fp16": {
+ "desc": null,
+ "value": false
+ },
+ "fsdp": {
+ "desc": null,
+ "value": []
+ },
+ "seed": {
+ "desc": null,
+ "value": 42
+ },
+ "tf32": {
+ "desc": null,
+ "value": false
+ },
+ "debug": {
+ "desc": null,
+ "value": []
+ },
+ "optim": {
+ "desc": null,
+ "value": "adamw_bnb_8bit"
+ },
+ "qlora": {
+ "desc": null,
+ "value": true
+ },
+ "top_k": {
+ "desc": null,
+ "value": 50
+ },
+ "top_p": {
+ "desc": null,
+ "value": 1
+ },
+ "_wandb": {
+ "desc": null,
+ "value": {
+ "m": [
+ {
+ "1": "train/global_step",
+ "6": [
+ 3
+ ]
+ },
+ {
+ "1": "train/loss",
+ "5": 1,
+ "6": [
+ 1
+ ]
+ },
+ {
+ "1": "train/grad_norm",
+ "5": 1,
+ "6": [
+ 1
+ ]
+ },
+ {
+ "1": "train/learning_rate",
+ "5": 1,
+ "6": [
+ 1
+ ]
+ },
+ {
+ "1": "train/epoch",
+ "5": 1,
+ "6": [
+ 1
+ ]
+ },
+ {
+ "1": "eval/loss",
+ "5": 1,
+ "6": [
+ 1
+ ]
+ },
+ {
+ "1": "eval/runtime",
+ "5": 1,
+ "6": [
+ 1
+ ]
+ },
+ {
+ "1": "eval/samples_per_second",
+ "5": 1,
+ "6": [
+ 1
+ ]
+ },
+ {
+ "1": "eval/steps_per_second",
+ "5": 1,
+ "6": [
+ 1
+ ]
+ }
+ ],
+ "t": {
+ "1": [
+ 1,
+ 5,
+ 11,
+ 49,
+ 51,
+ 53,
+ 55,
+ 71,
+ 84,
+ 98,
+ 99,
+ 100,
+ 105
+ ],
+ "2": [
+ 1,
+ 5,
+ 11,
+ 49,
+ 51,
+ 53,
+ 55,
+ 71,
+ 84,
+ 98,
+ 99,
+ 100,
+ 105
+ ],
+ "3": [
+ 3,
+ 7,
+ 23
+ ],
+ "4": "3.10.13",
+ "5": "0.16.4",
+ "6": "4.39.0.dev0",
+ "8": [
+ 5
+ ],
+ "9": {
+ "1": "transformers_trainer"
+ },
+ "13": "linux-x86_64"
+ },
+ "framework": "huggingface",
+ "start_time": 1711016068,
+ "cli_version": "0.16.4",
+ "is_jupyter_run": false,
+ "python_version": "3.10.13",
+ "is_kaggle_kernel": false,
+ "huggingface_version": "4.39.0.dev0"
+ }
+ },
+ "prefix": {
+ "desc": null,
+ "value": null
+ },
+ "do_eval": {
+ "desc": null,
+ "value": true
+ },
+ "no_cuda": {
+ "desc": null,
+ "value": false
+ },
+ "use_cpu": {
+ "desc": null,
+ "value": false
+ },
+ "do_train": {
+ "desc": null,
+ "value": false
+ },
+ "head_dim": {
+ "desc": null,
+ "value": 256
+ },
+ "id2label": {
+ "desc": null,
+ "value": {
+ "0": "LABEL_0",
+ "1": "LABEL_1"
+ }
+ },
+ "label2id": {
+ "desc": null,
+ "value": {
+ "LABEL_0": 0,
+ "LABEL_1": 1
+ }
+ },
+ "run_name": {
+ "desc": null,
+ "value": "./out"
+ },
+ "use_ipex": {
+ "desc": null,
+ "value": false
+ },
+ "adafactor": {
+ "desc": null,
+ "value": false
+ },
+ "data_seed": {
+ "desc": null,
+ "value": null
+ },
+ "deepspeed": {
+ "desc": null,
+ "value": "deepspeed_configs/zero1.json"
+ },
+ "do_sample": {
+ "desc": null,
+ "value": false
+ },
+ "hub_token": {
+ "desc": null,
+ },
+ "log_level": {
+ "desc": null,
+ "value": "passive"
+ },
+ "max_steps": {
+ "desc": null,
+ "value": -1
+ },
+ "num_beams": {
+ "desc": null,
+ "value": 1
+ },
+ "ray_scope": {
+ "desc": null,
+ "value": "last"
+ },
+ "report_to": {
+ "desc": null,
+ "value": [
+ "wandb"
+ ]
+ },
+ "typical_p": {
+ "desc": null,
+ "value": 1
+ },
+ "use_cache": {
+ "desc": null,
+ "value": false
+ },
+ "adam_beta1": {
+ "desc": null,
+ "value": 0.9
+ },
+ "adam_beta2": {
+ "desc": null,
+ "value": 0.999
+ },
+ "do_predict": {
+ "desc": null,
+ "value": false
+ },
+ "eval_delay": {
+ "desc": null,
+ "value": 0
+ },
+ "eval_steps": {
+ "desc": null,
+ "value": 0.03125
+ },
+ "hidden_act": {
+ "desc": null,
+ "value": "gelu"
+ },
+ "is_decoder": {
+ "desc": null,
+ "value": false
+ },
+ "local_rank": {
+ "desc": null,
+ "value": 0
+ },
+ "max_length": {
+ "desc": null,
+ "value": 20
+ },
+ "min_length": {
+ "desc": null,
+ "value": 0
+ },
+ "model_type": {
+ "desc": null,
+ "value": "gemma"
+ },
+ "optim_args": {
+ "desc": null,
+ "value": null
+ },
+ "orpo_alpha": {
+ "desc": null,
+ "value": null
+ },
+ "output_dir": {
+ "desc": null,
+ "value": "./out"
+ },
+ "past_index": {
+ "desc": null,
+ "value": -1
+ },
+ "rope_theta": {
+ "desc": null,
+ "value": 10000
+ },
+ "save_steps": {
+ "desc": null,
+ "value": 0.125
+ },
+ "vocab_size": {
+ "desc": null,
+ "value": 256000
+ },
+ "bench_split": {
+ "desc": null,
+ "value": "eval"
+ },
+ "ddp_backend": {
+ "desc": null,
+ "value": null
+ },
+ "ddp_timeout": {
+ "desc": null,
+ "value": 1800
+ },
+ "fsdp_config": {
+ "desc": null,
+ "value": {
+ "xla": false,
+ "xla_fsdp_v2": false,
+ "min_num_params": 0,
+ "xla_fsdp_grad_ckpt": false
+ }
+ },
+ "hidden_size": {
+ "desc": null,
+ "value": 2048
+ },
+ "label_names": {
+ "desc": null,
+ "value": null
+ },
+ "logging_dir": {
+ "desc": null,
+ "value": "./out/runs/Mar21_10-14-24_8205afe3ecd2"
+ },
+ "pretraining": {
+ "desc": null,
+ "value": false
+ },
+ "push_to_hub": {
+ "desc": null,
+ "value": false
+ },
+ "return_dict": {
+ "desc": null,
+ "value": true
+ },
+ "temperature": {
+ "desc": null,
+ "value": 1
+ },
+ "torch_dtype": {
+ "desc": null,
+ "value": "bfloat16"
+ },
+ "torchdynamo": {
+ "desc": null,
+ "value": null
+ },
+ "torchscript": {
+ "desc": null,
+ "value": false
+ },
+ "adam_epsilon": {
+ "desc": null,
+ "value": 1e-8
+ },
+ "bos_token_id": {
+ "desc": null,
+ "value": 2
+ },
+ "disable_tqdm": {
+ "desc": null,
+ "value": false
+ },
+ "eos_token_id": {
+ "desc": null,
+ "value": 1
+ },
+ "fp16_backend": {
+ "desc": null,
+ "value": "auto"
+ },
+ "hub_model_id": {
+ "desc": null,
+ "value": null
+ },
+ "hub_strategy": {
+ "desc": null,
+ "value": "every_save"
+ },
+ "pad_token_id": {
+ "desc": null,
+ "value": 0
+ },
+ "problem_type": {
+ "desc": null,
+ "value": null
+ },
+ "pruned_heads": {
+ "desc": null,
+ "value": {}
+ },
+ "relora_steps": {
+ "desc": null,
+ "value": null
+ },
+ "rms_norm_eps": {
+ "desc": null,
+ "value": 0.000001
+ },
+ "rope_scaling": {
+ "desc": null,
+ "value": null
+ },
+ "sep_token_id": {
+ "desc": null,
+ "value": null
+ },
+ "use_bfloat16": {
+ "desc": null,
+ "value": false
+ },
+ "warmup_ratio": {
+ "desc": null,
+ "value": 0
+ },
+ "warmup_steps": {
+ "desc": null,
+ "value": 3135
+ },
+ "weight_decay": {
+ "desc": null,
+ "value": 0
+ },
+ "_name_or_path": {
+ "desc": null,
+ "value": "dvdmrs09/gemma2b-train"
+ },
+ "architectures": {
+ "desc": null,
+ "value": [
+ "GemmaForCausalLM"
+ ]
+ },
+ "bad_words_ids": {
+ "desc": null,
+ "value": null
+ },
+ "bench_dataset": {
+ "desc": null,
+ "value": "pharaouk/dharma-1/dharma_1_mini.json"
+ },
+ "do_bench_eval": {
+ "desc": null,
+ "value": false
+ },
+ "jit_mode_eval": {
+ "desc": null,
+ "value": false
+ },
+ "learning_rate": {
+ "desc": null,
+ "value": 0.0002
+ },
+ "logging_steps": {
+ "desc": null,
+ "value": 1
+ },
+ "max_grad_norm": {
+ "desc": null,
+ "value": 1
+ },
+ "mp_parameters": {
+ "desc": null,
+ "value": ""
+ },
+ "output_scores": {
+ "desc": null,
+ "value": false
+ },
+ "save_strategy": {
+ "desc": null,
+ "value": "steps"
+ },
+ "split_batches": {
+ "desc": null,
+ "value": null
+ },
+ "torch_compile": {
+ "desc": null,
+ "value": false
+ },
+ "tpu_num_cores": {
+ "desc": null,
+ "value": null
+ },
+ "attention_bias": {
+ "desc": null,
+ "value": false
+ },
+ "bf16_full_eval": {
+ "desc": null,
+ "value": false
+ },
+ "early_stopping": {
+ "desc": null,
+ "value": false
+ },
+ "fp16_full_eval": {
+ "desc": null,
+ "value": false
+ },
+ "fp16_opt_level": {
+ "desc": null,
+ "value": "O1"
+ },
+ "length_penalty": {
+ "desc": null,
+ "value": 1
+ },
+ "max_seq_length": {
+ "desc": null,
+ "value": 4096
+ },
+ "sample_packing": {
+ "desc": null,
+ "value": false
+ },
+ "tf_legacy_loss": {
+ "desc": null,
+ "value": false
+ },
+ "use_mps_device": {
+ "desc": null,
+ "value": false
+ },
+ "finetuning_task": {
+ "desc": null,
+ "value": null
+ },
+ "group_by_length": {
+ "desc": null,
+ "value": false
+ },
+ "hub_always_push": {
+ "desc": null,
+ "value": false
+ },
+ "num_beam_groups": {
+ "desc": null,
+ "value": 1
+ },
+ "save_only_model": {
+ "desc": null,
+ "value": false
+ },
+ "suppress_tokens": {
+ "desc": null,
+ "value": null
+ },
+ "tokenizer_class": {
+ "desc": null,
+ "value": null
+ },
+ "dispatch_batches": {
+ "desc": null,
+ "value": null
+ },
+ "full_determinism": {
+ "desc": null,
+ "value": false
+ },
+ "hub_private_repo": {
+ "desc": null,
+ "value": false
+ },
+ "ignore_data_skip": {
+ "desc": null,
+ "value": false
+ },
+ "log_on_each_node": {
+ "desc": null,
+ "value": true
+ },
+ "logging_strategy": {
+ "desc": null,
+ "value": "steps"
+ },
+ "num_train_epochs": {
+ "desc": null,
+ "value": 8
+ },
+ "save_safetensors": {
+ "desc": null,
+ "value": true
+ },
+ "save_total_limit": {
+ "desc": null,
+ "value": 4
+ },
+ "attention_dropout": {
+ "desc": null,
+ "value": 0
+ },
+ "ddp_bucket_cap_mb": {
+ "desc": null,
+ "value": null
+ },
+ "diversity_penalty": {
+ "desc": null,
+ "value": 0
+ },
+ "do_causal_lm_eval": {
+ "desc": null,
+ "value": false
+ },
+ "greater_is_better": {
+ "desc": null,
+ "value": false
+ },
+ "initializer_range": {
+ "desc": null,
+ "value": 0.02
+ },
+ "intermediate_size": {
+ "desc": null,
+ "value": 16384
+ },
+ "log_level_replica": {
+ "desc": null,
+ "value": "warning"
+ },
+ "loraplus_lr_ratio": {
+ "desc": null,
+ "value": null
+ },
+ "lr_scheduler_type": {
+ "desc": null,
+ "value": "cosine"
+ },
+ "max_bench_samples": {
+ "desc": null,
+ "value": null
+ },
+ "num_hidden_layers": {
+ "desc": null,
+ "value": 18
+ },
+ "output_attentions": {
+ "desc": null,
+ "value": false
+ },
+ "push_to_hub_token": {
+ "desc": null,
+ "value": ""
+ },
+ "save_on_each_node": {
+ "desc": null,
+ "value": false
+ },
+ "tpu_metrics_debug": {
+ "desc": null,
+ "value": false
+ },
+ "accelerator_config": {
+ "desc": null,
+ "value": {
+ "even_batches": true,
+ "split_batches": false,
+ "dispatch_batches": null,
+ "use_seedable_sampler": true
+ }
+ },
+ "is_encoder_decoder": {
+ "desc": null,
+ "value": false
+ },
+ "length_column_name": {
+ "desc": null,
+ "value": "length"
+ },
+ "logging_first_step": {
+ "desc": null,
+ "value": false
+ },
+ "relora_prune_ratio": {
+ "desc": null,
+ "value": 0.9
+ },
+ "repetition_penalty": {
+ "desc": null,
+ "value": 1
+ },
+ "torch_compile_mode": {
+ "desc": null,
+ "value": null
+ },
+ "add_cross_attention": {
+ "desc": null,
+ "value": false
+ },
+ "cosine_min_lr_ratio": {
+ "desc": null,
+ "value": null
+ },
+ "eval_sample_packing": {
+ "desc": null,
+ "value": false
+ },
+ "evaluation_strategy": {
+ "desc": null,
+ "value": "steps"
+ },
+ "forced_bos_token_id": {
+ "desc": null,
+ "value": null
+ },
+ "forced_eos_token_id": {
+ "desc": null,
+ "value": null
+ },
+ "fsdp_min_num_params": {
+ "desc": null,
+ "value": 0
+ },
+ "lr_quadratic_warmup": {
+ "desc": null,
+ "value": false
+ },
+ "lr_scheduler_kwargs": {
+ "desc": null,
+ "value": {}
+ },
+ "neftune_noise_alpha": {
+ "desc": null,
+ "value": null
+ },
+ "num_attention_heads": {
+ "desc": null,
+ "value": 8
+ },
+ "num_key_value_heads": {
+ "desc": null,
+ "value": 1
+ },
+ "quantization_config": {
+ "desc": null,
+ "value": {
+ "load_in_4bit": true,
+ "load_in_8bit": false,
+ "quant_method": "QuantizationMethod.BITS_AND_BYTES",
+ "_load_in_4bit": true,
+ "_load_in_8bit": false,
+ "llm_int8_threshold": 6,
+ "bnb_4bit_quant_type": "nf4",
+ "llm_int8_skip_modules": null,
+ "bnb_4bit_compute_dtype": "bfloat16",
+ "bnb_4bit_quant_storage": "uint8",
+ "llm_int8_has_fp16_weight": false,
+ "bnb_4bit_use_double_quant": true,
+ "llm_int8_enable_fp32_cpu_offload": false
+ }
+ },
+ "relora_anneal_steps": {
+ "desc": null,
+ "value": null
+ },
+ "relora_warmup_steps": {
+ "desc": null,
+ "value": null
+ },
+ "skip_memory_metrics": {
+ "desc": null,
+ "value": true
+ },
+ "tie_encoder_decoder": {
+ "desc": null,
+ "value": false
+ },
+ "tie_word_embeddings": {
+ "desc": null,
+ "value": true
+ },
+ "auto_find_batch_size": {
+ "desc": null,
+ "value": false
+ },
+ "bench_source_max_len": {
+ "desc": null,
+ "value": 2048
+ },
+ "dataloader_drop_last": {
+ "desc": null,
+ "value": false
+ },
+ "no_repeat_ngram_size": {
+ "desc": null,
+ "value": 0
+ },
+ "num_return_sequences": {
+ "desc": null,
+ "value": 1
+ },
+ "optim_target_modules": {
+ "desc": null,
+ "value": null
+ },
+ "output_hidden_states": {
+ "desc": null,
+ "value": false
+ },
+ "overwrite_output_dir": {
+ "desc": null,
+ "value": false
+ },
+ "prediction_loss_only": {
+ "desc": null,
+ "value": false
+ },
+ "push_to_hub_model_id": {
+ "desc": null,
+ "value": null
+ },
+ "task_specific_params": {
+ "desc": null,
+ "value": null
+ },
+ "transformers_version": {
+ "desc": null,
+ "value": "4.39.0.dev0"
+ },
+ "begin_suppress_tokens": {
+ "desc": null,
+ "value": null
+ },
+ "dataloader_pin_memory": {
+ "desc": null,
+ "value": true
+ },
+ "ddp_broadcast_buffers": {
+ "desc": null,
+ "value": null
+ },
+ "loraplus_lr_embedding": {
+ "desc": null,
+ "value": null
+ },
+ "metric_for_best_model": {
+ "desc": null,
+ "value": "loss"
+ },
+ "remove_invalid_values": {
+ "desc": null,
+ "value": false
+ },
+ "remove_unused_columns": {
+ "desc": null,
+ "value": true
+ },
+ "torch_compile_backend": {
+ "desc": null,
+ "value": null
+ },
+ "dataloader_num_workers": {
+ "desc": null,
+ "value": 0
+ },
+ "decoder_start_token_id": {
+ "desc": null,
+ "value": null
+ },
+ "gradient_checkpointing": {
+ "desc": null,
+ "value": true
+ },
+ "half_precision_backend": {
+ "desc": null,
+ "value": "auto"
+ },
+ "label_smoothing_factor": {
+ "desc": null,
+ "value": 0
+ },
+ "load_best_model_at_end": {
+ "desc": null,
+ "value": true
+ },
+ "logging_nan_inf_filter": {
+ "desc": null,
+ "value": true
+ },
+ "multipack_real_batches": {
+ "desc": null,
+ "value": false
+ },
+ "resume_from_checkpoint": {
+ "desc": null,
+ "value": null
+ },
+ "chunk_size_feed_forward": {
+ "desc": null,
+ "value": 0
+ },
+ "eval_accumulation_steps": {
+ "desc": null,
+ "value": 3
+ },
+ "max_position_embeddings": {
+ "desc": null,
+ "value": 8192
+ },
+ "per_gpu_eval_batch_size": {
+ "desc": null,
+ "value": null
+ },
+ "return_dict_in_generate": {
+ "desc": null,
+ "value": false
+ },
+ "cosine_constant_lr_ratio": {
+ "desc": null,
+ "value": null
+ },
+ "per_gpu_train_batch_size": {
+ "desc": null,
+ "value": null
+ },
+ "push_to_hub_organization": {
+ "desc": null,
+ "value": null
+ },
+ "include_tokens_per_second": {
+ "desc": null,
+ "value": false
+ },
+ "sample_packing_efficiency": {
+ "desc": null,
+ "value": 1
+ },
+ "dataloader_prefetch_factor": {
+ "desc": null,
+ "value": null
+ },
+ "ddp_find_unused_parameters": {
+ "desc": null,
+ "value": false
+ },
+ "include_inputs_for_metrics": {
+ "desc": null,
+ "value": false
+ },
+ "per_device_eval_batch_size": {
+ "desc": null,
+ "value": 2
+ },
+ "use_legacy_prediction_loop": {
+ "desc": null,
+ "value": false
+ },
+ "cross_attention_hidden_size": {
+ "desc": null,
+ "value": null
+ },
+ "gradient_accumulation_steps": {
+ "desc": null,
+ "value": 3
+ },
+ "per_device_train_batch_size": {
+ "desc": null,
+ "value": 2
+ },
+ "encoder_no_repeat_ngram_size": {
+ "desc": null,
+ "value": 0
+ },
+ "dataloader_persistent_workers": {
+ "desc": null,
+ "value": false
+ },
+ "gradient_checkpointing_kwargs": {
+ "desc": null,
+ "value": {
+ "use_reentrant": true
+ }
+ },
+ "include_num_input_tokens_seen": {
+ "desc": null,
+ "value": false
+ },
+ "exponential_decay_length_penalty": {
+ "desc": null,
+ "value": null
+ },
+ "sample_packing_seq_len_multiplier": {
+ "desc": null,
+ "value": 2
+ },
+ "fsdp_transformer_layer_cls_to_wrap": {
+ "desc": null,
+ "value": null
+ }
+}
+### Model Architecture and Objective
+
+## Citation [optional]
+
+
+## Glossary [optional]
+### Framework versions
+
+- PEFT 0.9.0
diff --git a/checkpoint-448/adapter_config.json b/checkpoint-448/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a11027e1f94ce0414cb1f976775603b02ca0a82
--- /dev/null
+++ b/checkpoint-448/adapter_config.json
@@ -0,0 +1,33 @@
+{
+ "alpha_pattern": {},
+ "auto_mapping": null,
+ "base_model_name_or_path": "google/gemma-2b",
+ "bias": "none",
+ "fan_in_fan_out": null,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 16,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "v_proj",
+ "down_proj",
+ "gate_proj",
+ "k_proj",
+ "o_proj"
+ ],
+ "task_type": "CAUSAL_LM",
+ "use_dora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-448/adapter_model.safetensors b/checkpoint-448/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0f3c1a98c474666070807683551bb113ba53426d
--- /dev/null
+++ b/checkpoint-448/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f99a127576ffaecb34de3d4edc89d1949ba3b1679dc1ed7fd2842d82eb9455bf
+size 78480320
diff --git a/checkpoint-448/global_step448/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-448/global_step448/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5701a82c7f4087e3f36d9fe133c2088e2862b707
--- /dev/null
+++ b/checkpoint-448/global_step448/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43c24988891062476de7382ed15e90fbad5418225d862ceb6fd88e9480502287
+size 58886928
diff --git a/checkpoint-448/global_step448/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-448/global_step448/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e0ca1028d1c65b5c69858bebc19908f2fb45a4d
--- /dev/null
+++ b/checkpoint-448/global_step448/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99ab332d1739e2f3c68850dc96ff616c41dd510d174f3147ed26532e7c4514af
+size 58885968
diff --git a/checkpoint-448/global_step448/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/checkpoint-448/global_step448/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0c5e4955c0a4f5ca489ac4feff5b098dd07c47ff
--- /dev/null
+++ b/checkpoint-448/global_step448/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e7dc29b4b32f6ec9dcb5fc8be4fa57f424d8f9f2b0294472a3334a37fd7a5fe
+size 58886992
diff --git a/checkpoint-448/global_step448/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/checkpoint-448/global_step448/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bef50e0b9ec52b2993deb0e6e2c33d2d20422b62
--- /dev/null
+++ b/checkpoint-448/global_step448/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:274561b107cc73f237a78a2781ca85f1bd99e47cd797182bbc3c7bf629069a49
+size 58886032
diff --git a/checkpoint-448/global_step448/mp_rank_00_model_states.pt b/checkpoint-448/global_step448/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ce480109d44c6ddf411ceff715548d2c21919e68
--- /dev/null
+++ b/checkpoint-448/global_step448/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c28a01f237249400db81971db6adfc9c23deb3e84e97d8ec17b329ef0d45dbf
+size 1159049922
diff --git a/checkpoint-448/latest b/checkpoint-448/latest
new file mode 100644
index 0000000000000000000000000000000000000000..6c83691d1f18f1aa59c0994e76f1e0d010c88273
--- /dev/null
+++ b/checkpoint-448/latest
@@ -0,0 +1 @@
+global_step448
\ No newline at end of file
diff --git a/checkpoint-448/rng_state_0.pth b/checkpoint-448/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..07538ca228f8bf27ad2e09c1b0b78cb8fb52e1bd
--- /dev/null
+++ b/checkpoint-448/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b355525fc0c81100cf2575a311c308531a030d69aa689ab177e589ba734ba26f
+size 15024
diff --git a/checkpoint-448/rng_state_1.pth b/checkpoint-448/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a4ecc3647c3912fe6e0b6cb5ae16c49136ed4b05
--- /dev/null
+++ b/checkpoint-448/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b43ae95006a17e1c105f80d795d47c96c50c4d3de9b0fa179869e47dda9a2b7
+size 15024
diff --git a/checkpoint-448/rng_state_2.pth b/checkpoint-448/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..77ebab12083b39188ce06023d589ff8418b21a8e
--- /dev/null
+++ b/checkpoint-448/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c56cc8de2cb76701e40dba1d6536107c0b5c90cf594d9db76a94a2b09f972e86
+size 15024
diff --git a/checkpoint-448/rng_state_3.pth b/checkpoint-448/rng_state_3.pth
new file mode 100644
index 0000000000000000000000000000000000000000..82e6d99eee33faadfe671b0d49de0c9991fd489b
--- /dev/null
+++ b/checkpoint-448/rng_state_3.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fc458f140d74069a4aa7df11c36e627bc9f362022588000b636afe9124026e7
+size 15024
diff --git a/checkpoint-448/scheduler.pt b/checkpoint-448/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9a92403b8c8ccb7f0b39932e09c835b8e10df10
--- /dev/null
+++ b/checkpoint-448/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0f312f7dfc21d940ab17e42a9ddfcca8440bfc594bbdb7174eb9bb867dd848d
+size 1064
diff --git a/checkpoint-448/trainer_state.json b/checkpoint-448/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..c184a2783d7b4275aafa7cfe890c28da9206b508
--- /dev/null
+++ b/checkpoint-448/trainer_state.json
@@ -0,0 +1,3293 @@
+{
+ "best_metric": 1.203959345817566,
+ "best_model_checkpoint": "./out/checkpoint-112",
+ "epoch": 3.9881305637982196,
+ "eval_steps": 28,
+ "global_step": 448,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.01,
+ "grad_norm": 4.313233023002325,
+ "learning_rate": 1.8181818181818182e-05,
+ "loss": 1.9528,
+ "step": 1
+ },
+ {
+ "epoch": 0.01,
+ "eval_loss": 2.1875686645507812,
+ "eval_runtime": 12.8608,
+ "eval_samples_per_second": 23.327,
+ "eval_steps_per_second": 2.955,
+ "step": 1
+ },
+ {
+ "epoch": 0.02,
+ "grad_norm": 4.039172290955229,
+ "learning_rate": 3.6363636363636364e-05,
+ "loss": 1.8358,
+ "step": 2
+ },
+ {
+ "epoch": 0.03,
+ "grad_norm": 4.504705512003857,
+ "learning_rate": 5.4545454545454546e-05,
+ "loss": 2.0207,
+ "step": 3
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 4.591862504847867,
+ "learning_rate": 7.272727272727273e-05,
+ "loss": 1.979,
+ "step": 4
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 3.812893581399005,
+ "learning_rate": 9.090909090909092e-05,
+ "loss": 1.8356,
+ "step": 5
+ },
+ {
+ "epoch": 0.05,
+ "grad_norm": 0.42886752872747064,
+ "learning_rate": 0.00010909090909090909,
+ "loss": 1.6722,
+ "step": 6
+ },
+ {
+ "epoch": 0.06,
+ "grad_norm": 0.22497294481851865,
+ "learning_rate": 0.00012727272727272728,
+ "loss": 1.6711,
+ "step": 7
+ },
+ {
+ "epoch": 0.07,
+ "grad_norm": 0.20955259847301927,
+ "learning_rate": 0.00014545454545454546,
+ "loss": 1.8546,
+ "step": 8
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 0.2200095325539683,
+ "learning_rate": 0.00016363636363636366,
+ "loss": 1.7538,
+ "step": 9
+ },
+ {
+ "epoch": 0.09,
+ "grad_norm": 0.19187339879899318,
+ "learning_rate": 0.00018181818181818183,
+ "loss": 1.6137,
+ "step": 10
+ },
+ {
+ "epoch": 0.1,
+ "grad_norm": 0.2113395673717837,
+ "learning_rate": 0.0002,
+ "loss": 1.5225,
+ "step": 11
+ },
+ {
+ "epoch": 0.11,
+ "grad_norm": 0.17673768408382828,
+ "learning_rate": 0.00019999741592564903,
+ "loss": 1.5303,
+ "step": 12
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 0.24120852820548402,
+ "learning_rate": 0.00019998966383614488,
+ "loss": 1.5089,
+ "step": 13
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 0.3089489160535682,
+ "learning_rate": 0.00019997674413212708,
+ "loss": 1.4525,
+ "step": 14
+ },
+ {
+ "epoch": 0.13,
+ "grad_norm": 0.2656143410731927,
+ "learning_rate": 0.00019995865748130516,
+ "loss": 1.4648,
+ "step": 15
+ },
+ {
+ "epoch": 0.14,
+ "grad_norm": 3.769410316227205,
+ "learning_rate": 0.0001999354048184241,
+ "loss": 1.3439,
+ "step": 16
+ },
+ {
+ "epoch": 0.15,
+ "grad_norm": 0.32102180658823753,
+ "learning_rate": 0.00019990698734521613,
+ "loss": 1.4644,
+ "step": 17
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 0.22094428128919438,
+ "learning_rate": 0.0001998734065303385,
+ "loss": 1.1927,
+ "step": 18
+ },
+ {
+ "epoch": 0.17,
+ "grad_norm": 0.22344487218098863,
+ "learning_rate": 0.00019983466410929764,
+ "loss": 1.2916,
+ "step": 19
+ },
+ {
+ "epoch": 0.18,
+ "grad_norm": 0.25036262498479456,
+ "learning_rate": 0.0001997907620843595,
+ "loss": 1.2982,
+ "step": 20
+ },
+ {
+ "epoch": 0.19,
+ "grad_norm": 0.22671119151539426,
+ "learning_rate": 0.00019974170272444604,
+ "loss": 1.2146,
+ "step": 21
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 0.259249080403425,
+ "learning_rate": 0.00019968748856501788,
+ "loss": 1.2072,
+ "step": 22
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 0.23538477651406017,
+ "learning_rate": 0.00019962812240794343,
+ "loss": 1.3281,
+ "step": 23
+ },
+ {
+ "epoch": 0.21,
+ "grad_norm": 0.2659115087625978,
+ "learning_rate": 0.000199563607321354,
+ "loss": 1.1396,
+ "step": 24
+ },
+ {
+ "epoch": 0.22,
+ "grad_norm": 0.23617264858854836,
+ "learning_rate": 0.0001994939466394851,
+ "loss": 1.1389,
+ "step": 25
+ },
+ {
+ "epoch": 0.23,
+ "grad_norm": 0.20514227454180176,
+ "learning_rate": 0.00019941914396250446,
+ "loss": 1.249,
+ "step": 26
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.19660894225830144,
+ "learning_rate": 0.00019933920315632557,
+ "loss": 1.1776,
+ "step": 27
+ },
+ {
+ "epoch": 0.25,
+ "grad_norm": 0.2067663909729571,
+ "learning_rate": 0.00019925412835240826,
+ "loss": 1.1327,
+ "step": 28
+ },
+ {
+ "epoch": 0.25,
+ "eval_loss": 1.2991960048675537,
+ "eval_runtime": 13.153,
+ "eval_samples_per_second": 22.808,
+ "eval_steps_per_second": 2.889,
+ "step": 28
+ },
+ {
+ "epoch": 0.26,
+ "grad_norm": 0.1816588361901526,
+ "learning_rate": 0.0001991639239475448,
+ "loss": 1.1247,
+ "step": 29
+ },
+ {
+ "epoch": 0.27,
+ "grad_norm": 0.19626955153633807,
+ "learning_rate": 0.00019906859460363307,
+ "loss": 1.1212,
+ "step": 30
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 0.21084275590405852,
+ "learning_rate": 0.00019896814524743528,
+ "loss": 0.9927,
+ "step": 31
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 0.16560054949456768,
+ "learning_rate": 0.0001988625810703235,
+ "loss": 1.1249,
+ "step": 32
+ },
+ {
+ "epoch": 0.29,
+ "grad_norm": 0.14950879528294536,
+ "learning_rate": 0.0001987519075280114,
+ "loss": 1.1401,
+ "step": 33
+ },
+ {
+ "epoch": 0.3,
+ "grad_norm": 0.1777966882651237,
+ "learning_rate": 0.00019863613034027224,
+ "loss": 1.0769,
+ "step": 34
+ },
+ {
+ "epoch": 0.31,
+ "grad_norm": 0.1480537272052743,
+ "learning_rate": 0.00019851525549064323,
+ "loss": 1.0686,
+ "step": 35
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.16911906750319078,
+ "learning_rate": 0.00019838928922611632,
+ "loss": 1.0253,
+ "step": 36
+ },
+ {
+ "epoch": 0.33,
+ "grad_norm": 0.15987682972555176,
+ "learning_rate": 0.00019825823805681543,
+ "loss": 1.0609,
+ "step": 37
+ },
+ {
+ "epoch": 0.34,
+ "grad_norm": 0.15757332939676763,
+ "learning_rate": 0.0001981221087556598,
+ "loss": 1.1086,
+ "step": 38
+ },
+ {
+ "epoch": 0.35,
+ "grad_norm": 0.13201845744757537,
+ "learning_rate": 0.00019798090835801418,
+ "loss": 1.073,
+ "step": 39
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 0.12544508015984754,
+ "learning_rate": 0.00019783464416132506,
+ "loss": 1.0633,
+ "step": 40
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 0.14645820383886451,
+ "learning_rate": 0.00019768332372474366,
+ "loss": 1.0653,
+ "step": 41
+ },
+ {
+ "epoch": 0.37,
+ "grad_norm": 0.14814101902137117,
+ "learning_rate": 0.00019752695486873517,
+ "loss": 1.0937,
+ "step": 42
+ },
+ {
+ "epoch": 0.38,
+ "grad_norm": 0.13888915595055443,
+ "learning_rate": 0.00019736554567467452,
+ "loss": 1.0462,
+ "step": 43
+ },
+ {
+ "epoch": 0.39,
+ "grad_norm": 0.13185349806639524,
+ "learning_rate": 0.00019719910448442893,
+ "loss": 1.2177,
+ "step": 44
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.15271046712350847,
+ "learning_rate": 0.00019702763989992662,
+ "loss": 1.0237,
+ "step": 45
+ },
+ {
+ "epoch": 0.41,
+ "grad_norm": 0.17053588557430902,
+ "learning_rate": 0.00019685116078271223,
+ "loss": 1.0038,
+ "step": 46
+ },
+ {
+ "epoch": 0.42,
+ "grad_norm": 0.15641087356577812,
+ "learning_rate": 0.00019666967625348906,
+ "loss": 1.0886,
+ "step": 47
+ },
+ {
+ "epoch": 0.43,
+ "grad_norm": 0.1544028594191567,
+ "learning_rate": 0.00019648319569164736,
+ "loss": 1.1378,
+ "step": 48
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 0.14794885994140625,
+ "learning_rate": 0.00019629172873477995,
+ "loss": 1.1495,
+ "step": 49
+ },
+ {
+ "epoch": 0.45,
+ "grad_norm": 0.1577684884028266,
+ "learning_rate": 0.0001960952852781838,
+ "loss": 1.0782,
+ "step": 50
+ },
+ {
+ "epoch": 0.45,
+ "grad_norm": 0.15961044045091288,
+ "learning_rate": 0.0001958938754743489,
+ "loss": 1.0107,
+ "step": 51
+ },
+ {
+ "epoch": 0.46,
+ "grad_norm": 0.14486696586022083,
+ "learning_rate": 0.0001956875097324334,
+ "loss": 1.0494,
+ "step": 52
+ },
+ {
+ "epoch": 0.47,
+ "grad_norm": 0.14250413725518896,
+ "learning_rate": 0.00019547619871772574,
+ "loss": 1.039,
+ "step": 53
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.1196720279125328,
+ "learning_rate": 0.00019525995335109334,
+ "loss": 1.0966,
+ "step": 54
+ },
+ {
+ "epoch": 0.49,
+ "grad_norm": 0.14984795891635327,
+ "learning_rate": 0.0001950387848084183,
+ "loss": 1.0874,
+ "step": 55
+ },
+ {
+ "epoch": 0.5,
+ "grad_norm": 0.14891088442480416,
+ "learning_rate": 0.00019481270452001987,
+ "loss": 1.097,
+ "step": 56
+ },
+ {
+ "epoch": 0.5,
+ "eval_loss": 1.2264304161071777,
+ "eval_runtime": 13.2279,
+ "eval_samples_per_second": 22.679,
+ "eval_steps_per_second": 2.873,
+ "step": 56
+ },
+ {
+ "epoch": 0.51,
+ "grad_norm": 0.17814266552244534,
+ "learning_rate": 0.00019458172417006347,
+ "loss": 1.1372,
+ "step": 57
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 0.16125636132578247,
+ "learning_rate": 0.00019434585569595708,
+ "loss": 1.0623,
+ "step": 58
+ },
+ {
+ "epoch": 0.53,
+ "grad_norm": 0.15203437202125702,
+ "learning_rate": 0.00019410511128773418,
+ "loss": 1.0399,
+ "step": 59
+ },
+ {
+ "epoch": 0.53,
+ "grad_norm": 0.1677461135605213,
+ "learning_rate": 0.0001938595033874238,
+ "loss": 1.0884,
+ "step": 60
+ },
+ {
+ "epoch": 0.54,
+ "grad_norm": 0.13564559875683407,
+ "learning_rate": 0.0001936090446884074,
+ "loss": 1.0176,
+ "step": 61
+ },
+ {
+ "epoch": 0.55,
+ "grad_norm": 0.1521886500642157,
+ "learning_rate": 0.00019335374813476302,
+ "loss": 1.0146,
+ "step": 62
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.1410132122625916,
+ "learning_rate": 0.00019309362692059617,
+ "loss": 1.044,
+ "step": 63
+ },
+ {
+ "epoch": 0.57,
+ "grad_norm": 0.15237848179385577,
+ "learning_rate": 0.00019282869448935798,
+ "loss": 1.0354,
+ "step": 64
+ },
+ {
+ "epoch": 0.58,
+ "grad_norm": 0.13871660988504514,
+ "learning_rate": 0.00019255896453315052,
+ "loss": 1.0189,
+ "step": 65
+ },
+ {
+ "epoch": 0.59,
+ "grad_norm": 0.14863047478901453,
+ "learning_rate": 0.000192284450992019,
+ "loss": 1.0704,
+ "step": 66
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 0.13794806124403974,
+ "learning_rate": 0.0001920051680532314,
+ "loss": 1.0996,
+ "step": 67
+ },
+ {
+ "epoch": 0.61,
+ "grad_norm": 0.13030507705779365,
+ "learning_rate": 0.00019172113015054532,
+ "loss": 1.0015,
+ "step": 68
+ },
+ {
+ "epoch": 0.61,
+ "grad_norm": 0.15092494718902358,
+ "learning_rate": 0.0001914323519634619,
+ "loss": 1.0822,
+ "step": 69
+ },
+ {
+ "epoch": 0.62,
+ "grad_norm": 0.1350212989006066,
+ "learning_rate": 0.00019113884841646736,
+ "loss": 1.0197,
+ "step": 70
+ },
+ {
+ "epoch": 0.63,
+ "grad_norm": 0.18991168066586347,
+ "learning_rate": 0.00019084063467826137,
+ "loss": 1.046,
+ "step": 71
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.14884381774710187,
+ "learning_rate": 0.00019053772616097337,
+ "loss": 1.0346,
+ "step": 72
+ },
+ {
+ "epoch": 0.65,
+ "grad_norm": 0.15579311209945296,
+ "learning_rate": 0.000190230138519366,
+ "loss": 1.0505,
+ "step": 73
+ },
+ {
+ "epoch": 0.66,
+ "grad_norm": 0.16015337150592127,
+ "learning_rate": 0.000189917887650026,
+ "loss": 1.0504,
+ "step": 74
+ },
+ {
+ "epoch": 0.67,
+ "grad_norm": 0.1443969321518926,
+ "learning_rate": 0.00018960098969054255,
+ "loss": 1.0755,
+ "step": 75
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 0.15722162227095848,
+ "learning_rate": 0.00018927946101867347,
+ "loss": 1.0541,
+ "step": 76
+ },
+ {
+ "epoch": 0.69,
+ "grad_norm": 0.17009697584926559,
+ "learning_rate": 0.0001889533182514986,
+ "loss": 1.0231,
+ "step": 77
+ },
+ {
+ "epoch": 0.69,
+ "grad_norm": 0.1256822726781221,
+ "learning_rate": 0.0001886225782445612,
+ "loss": 0.8814,
+ "step": 78
+ },
+ {
+ "epoch": 0.7,
+ "grad_norm": 0.14019958069756655,
+ "learning_rate": 0.00018828725809099655,
+ "loss": 1.0277,
+ "step": 79
+ },
+ {
+ "epoch": 0.71,
+ "grad_norm": 0.17159459150063183,
+ "learning_rate": 0.0001879473751206489,
+ "loss": 0.9495,
+ "step": 80
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.146430011834186,
+ "learning_rate": 0.00018760294689917553,
+ "loss": 1.0598,
+ "step": 81
+ },
+ {
+ "epoch": 0.73,
+ "grad_norm": 0.16834256802992476,
+ "learning_rate": 0.00018725399122713912,
+ "loss": 1.0237,
+ "step": 82
+ },
+ {
+ "epoch": 0.74,
+ "grad_norm": 0.15663699267164208,
+ "learning_rate": 0.00018690052613908772,
+ "loss": 0.939,
+ "step": 83
+ },
+ {
+ "epoch": 0.75,
+ "grad_norm": 0.15655985150409854,
+ "learning_rate": 0.0001865425699026226,
+ "loss": 1.0302,
+ "step": 84
+ },
+ {
+ "epoch": 0.75,
+ "eval_loss": 1.2143030166625977,
+ "eval_runtime": 13.2387,
+ "eval_samples_per_second": 22.661,
+ "eval_steps_per_second": 2.87,
+ "step": 84
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 0.15273470110260864,
+ "learning_rate": 0.00018618014101745442,
+ "loss": 1.0127,
+ "step": 85
+ },
+ {
+ "epoch": 0.77,
+ "grad_norm": 0.1723243680259614,
+ "learning_rate": 0.0001858132582144469,
+ "loss": 0.9306,
+ "step": 86
+ },
+ {
+ "epoch": 0.77,
+ "grad_norm": 0.14747098547446996,
+ "learning_rate": 0.00018544194045464886,
+ "loss": 1.0073,
+ "step": 87
+ },
+ {
+ "epoch": 0.78,
+ "grad_norm": 0.17208333285514918,
+ "learning_rate": 0.00018506620692831428,
+ "loss": 1.0328,
+ "step": 88
+ },
+ {
+ "epoch": 0.79,
+ "grad_norm": 0.14918051024971962,
+ "learning_rate": 0.0001846860770539105,
+ "loss": 1.0022,
+ "step": 89
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.156315164090714,
+ "learning_rate": 0.00018430157047711474,
+ "loss": 1.0293,
+ "step": 90
+ },
+ {
+ "epoch": 0.81,
+ "grad_norm": 0.2013424548288477,
+ "learning_rate": 0.00018391270706979862,
+ "loss": 0.9395,
+ "step": 91
+ },
+ {
+ "epoch": 0.82,
+ "grad_norm": 0.17909726353002614,
+ "learning_rate": 0.00018351950692900126,
+ "loss": 0.9756,
+ "step": 92
+ },
+ {
+ "epoch": 0.83,
+ "grad_norm": 0.16939245158726288,
+ "learning_rate": 0.00018312199037589068,
+ "loss": 0.9576,
+ "step": 93
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 0.14685720680893694,
+ "learning_rate": 0.00018272017795471345,
+ "loss": 1.0045,
+ "step": 94
+ },
+ {
+ "epoch": 0.85,
+ "grad_norm": 0.17464839085505987,
+ "learning_rate": 0.000182314090431733,
+ "loss": 0.9862,
+ "step": 95
+ },
+ {
+ "epoch": 0.85,
+ "grad_norm": 0.16060904136932572,
+ "learning_rate": 0.00018190374879415632,
+ "loss": 1.0022,
+ "step": 96
+ },
+ {
+ "epoch": 0.86,
+ "grad_norm": 0.18715193350083867,
+ "learning_rate": 0.00018148917424904953,
+ "loss": 1.042,
+ "step": 97
+ },
+ {
+ "epoch": 0.87,
+ "grad_norm": 0.1675573400576595,
+ "learning_rate": 0.0001810703882222415,
+ "loss": 1.0047,
+ "step": 98
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.1871466286989249,
+ "learning_rate": 0.00018064741235721687,
+ "loss": 0.9834,
+ "step": 99
+ },
+ {
+ "epoch": 0.89,
+ "grad_norm": 0.17453934867565302,
+ "learning_rate": 0.00018022026851399737,
+ "loss": 0.9649,
+ "step": 100
+ },
+ {
+ "epoch": 0.9,
+ "grad_norm": 0.15960631507184767,
+ "learning_rate": 0.0001797889787680119,
+ "loss": 0.9673,
+ "step": 101
+ },
+ {
+ "epoch": 0.91,
+ "grad_norm": 0.17844936635366368,
+ "learning_rate": 0.00017935356540895597,
+ "loss": 1.0951,
+ "step": 102
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 0.16733018789000254,
+ "learning_rate": 0.00017891405093963938,
+ "loss": 0.9954,
+ "step": 103
+ },
+ {
+ "epoch": 0.93,
+ "grad_norm": 0.17305556075296993,
+ "learning_rate": 0.00017847045807482345,
+ "loss": 0.892,
+ "step": 104
+ },
+ {
+ "epoch": 0.93,
+ "grad_norm": 0.17197614099805034,
+ "learning_rate": 0.00017802280974004716,
+ "loss": 1.0494,
+ "step": 105
+ },
+ {
+ "epoch": 0.94,
+ "grad_norm": 0.18063836817127235,
+ "learning_rate": 0.000177571129070442,
+ "loss": 1.0264,
+ "step": 106
+ },
+ {
+ "epoch": 0.95,
+ "grad_norm": 0.14597707005699143,
+ "learning_rate": 0.00017711543940953668,
+ "loss": 0.9532,
+ "step": 107
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 0.1422048149465345,
+ "learning_rate": 0.00017665576430805053,
+ "loss": 0.97,
+ "step": 108
+ },
+ {
+ "epoch": 0.97,
+ "grad_norm": 0.18313914688655572,
+ "learning_rate": 0.0001761921275226763,
+ "loss": 0.9282,
+ "step": 109
+ },
+ {
+ "epoch": 0.98,
+ "grad_norm": 0.200679751171441,
+ "learning_rate": 0.00017572455301485249,
+ "loss": 1.0,
+ "step": 110
+ },
+ {
+ "epoch": 0.99,
+ "grad_norm": 0.17700985594898055,
+ "learning_rate": 0.00017525306494952498,
+ "loss": 1.0165,
+ "step": 111
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 0.19925777202726191,
+ "learning_rate": 0.0001747776876938981,
+ "loss": 1.0346,
+ "step": 112
+ },
+ {
+ "epoch": 1.0,
+ "eval_loss": 1.203959345817566,
+ "eval_runtime": 13.2547,
+ "eval_samples_per_second": 22.634,
+ "eval_steps_per_second": 2.867,
+ "step": 112
+ },
+ {
+ "epoch": 1.01,
+ "grad_norm": 0.1606469603473709,
+ "learning_rate": 0.00017429844581617532,
+ "loss": 0.9832,
+ "step": 113
+ },
+ {
+ "epoch": 1.01,
+ "grad_norm": 0.16403912763780054,
+ "learning_rate": 0.00017381536408428948,
+ "loss": 0.9346,
+ "step": 114
+ },
+ {
+ "epoch": 1.02,
+ "grad_norm": 0.1936046893744468,
+ "learning_rate": 0.00017332846746462288,
+ "loss": 0.9382,
+ "step": 115
+ },
+ {
+ "epoch": 1.03,
+ "grad_norm": 0.14250769247239573,
+ "learning_rate": 0.0001728377811207168,
+ "loss": 0.8914,
+ "step": 116
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.17889563599797687,
+ "learning_rate": 0.00017234333041197126,
+ "loss": 0.9736,
+ "step": 117
+ },
+ {
+ "epoch": 1.05,
+ "grad_norm": 0.20288960866045594,
+ "learning_rate": 0.00017184514089233405,
+ "loss": 0.8477,
+ "step": 118
+ },
+ {
+ "epoch": 1.06,
+ "grad_norm": 0.20926349930533472,
+ "learning_rate": 0.00017134323830898037,
+ "loss": 0.9933,
+ "step": 119
+ },
+ {
+ "epoch": 1.07,
+ "grad_norm": 0.21316934416499642,
+ "learning_rate": 0.00017083764860098205,
+ "loss": 0.9168,
+ "step": 120
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 0.21654320387312692,
+ "learning_rate": 0.0001703283978979671,
+ "loss": 0.9584,
+ "step": 121
+ },
+ {
+ "epoch": 1.09,
+ "grad_norm": 0.23789742308175463,
+ "learning_rate": 0.00016981551251876904,
+ "loss": 1.0298,
+ "step": 122
+ },
+ {
+ "epoch": 1.09,
+ "grad_norm": 0.16433271793469648,
+ "learning_rate": 0.00016929901897006698,
+ "loss": 0.8833,
+ "step": 123
+ },
+ {
+ "epoch": 1.1,
+ "grad_norm": 0.16908727866207868,
+ "learning_rate": 0.0001687789439450156,
+ "loss": 1.0675,
+ "step": 124
+ },
+ {
+ "epoch": 1.11,
+ "grad_norm": 0.1670067931363302,
+ "learning_rate": 0.00016825531432186543,
+ "loss": 0.9515,
+ "step": 125
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.17777465531550865,
+ "learning_rate": 0.00016772815716257412,
+ "loss": 0.8929,
+ "step": 126
+ },
+ {
+ "epoch": 1.13,
+ "grad_norm": 0.18442783204919333,
+ "learning_rate": 0.00016719749971140754,
+ "loss": 0.8388,
+ "step": 127
+ },
+ {
+ "epoch": 1.14,
+ "grad_norm": 0.19073362304284272,
+ "learning_rate": 0.0001666633693935319,
+ "loss": 0.9584,
+ "step": 128
+ },
+ {
+ "epoch": 1.15,
+ "grad_norm": 0.20189563405135308,
+ "learning_rate": 0.00016612579381359622,
+ "loss": 1.0264,
+ "step": 129
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 0.1694138210313381,
+ "learning_rate": 0.00016558480075430594,
+ "loss": 0.9592,
+ "step": 130
+ },
+ {
+ "epoch": 1.17,
+ "grad_norm": 0.19195382946787184,
+ "learning_rate": 0.00016504041817498678,
+ "loss": 0.974,
+ "step": 131
+ },
+ {
+ "epoch": 1.18,
+ "grad_norm": 0.20684215619155688,
+ "learning_rate": 0.00016449267421013994,
+ "loss": 0.8499,
+ "step": 132
+ },
+ {
+ "epoch": 1.18,
+ "grad_norm": 0.22003490429847744,
+ "learning_rate": 0.00016394159716798807,
+ "loss": 0.9659,
+ "step": 133
+ },
+ {
+ "epoch": 1.19,
+ "grad_norm": 0.21977918206745437,
+ "learning_rate": 0.00016338721552901212,
+ "loss": 0.9213,
+ "step": 134
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.2076993903333204,
+ "learning_rate": 0.0001628295579444796,
+ "loss": 0.8119,
+ "step": 135
+ },
+ {
+ "epoch": 1.21,
+ "grad_norm": 0.2001771499954729,
+ "learning_rate": 0.0001622686532349637,
+ "loss": 0.9183,
+ "step": 136
+ },
+ {
+ "epoch": 1.22,
+ "grad_norm": 0.18671550149366203,
+ "learning_rate": 0.00016170453038885394,
+ "loss": 0.8836,
+ "step": 137
+ },
+ {
+ "epoch": 1.23,
+ "grad_norm": 0.20867427207572573,
+ "learning_rate": 0.0001611372185608578,
+ "loss": 0.9964,
+ "step": 138
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 0.20035138443113176,
+ "learning_rate": 0.0001605667470704942,
+ "loss": 0.9209,
+ "step": 139
+ },
+ {
+ "epoch": 1.25,
+ "grad_norm": 0.22696612020505577,
+ "learning_rate": 0.0001599931454005781,
+ "loss": 1.0162,
+ "step": 140
+ },
+ {
+ "epoch": 1.25,
+ "eval_loss": 1.2188584804534912,
+ "eval_runtime": 13.249,
+ "eval_samples_per_second": 22.643,
+ "eval_steps_per_second": 2.868,
+ "step": 140
+ },
+ {
+ "epoch": 1.26,
+ "grad_norm": 0.21554353495018647,
+ "learning_rate": 0.00015941644319569665,
+ "loss": 1.0487,
+ "step": 141
+ },
+ {
+ "epoch": 1.26,
+ "grad_norm": 0.22894492131909072,
+ "learning_rate": 0.00015883667026067745,
+ "loss": 0.9352,
+ "step": 142
+ },
+ {
+ "epoch": 1.27,
+ "grad_norm": 0.19145184577172686,
+ "learning_rate": 0.00015825385655904788,
+ "loss": 0.8878,
+ "step": 143
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 0.22544664152936575,
+ "learning_rate": 0.00015766803221148673,
+ "loss": 1.0,
+ "step": 144
+ },
+ {
+ "epoch": 1.29,
+ "grad_norm": 0.26000661355557114,
+ "learning_rate": 0.00015707922749426737,
+ "loss": 0.9339,
+ "step": 145
+ },
+ {
+ "epoch": 1.3,
+ "grad_norm": 0.24433845134512236,
+ "learning_rate": 0.00015648747283769317,
+ "loss": 0.9474,
+ "step": 146
+ },
+ {
+ "epoch": 1.31,
+ "grad_norm": 0.21973931169609887,
+ "learning_rate": 0.00015589279882452476,
+ "loss": 0.9357,
+ "step": 147
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 0.23929008733305812,
+ "learning_rate": 0.0001552952361883994,
+ "loss": 0.9985,
+ "step": 148
+ },
+ {
+ "epoch": 1.33,
+ "grad_norm": 0.23431856747573573,
+ "learning_rate": 0.00015469481581224272,
+ "loss": 0.8913,
+ "step": 149
+ },
+ {
+ "epoch": 1.34,
+ "grad_norm": 0.2233543327912565,
+ "learning_rate": 0.00015409156872667258,
+ "loss": 0.9877,
+ "step": 150
+ },
+ {
+ "epoch": 1.34,
+ "grad_norm": 0.21281207674183256,
+ "learning_rate": 0.0001534855261083954,
+ "loss": 0.9071,
+ "step": 151
+ },
+ {
+ "epoch": 1.35,
+ "grad_norm": 0.20314832700152685,
+ "learning_rate": 0.00015287671927859494,
+ "loss": 0.9373,
+ "step": 152
+ },
+ {
+ "epoch": 1.36,
+ "grad_norm": 0.19648565819019825,
+ "learning_rate": 0.00015226517970131343,
+ "loss": 0.9469,
+ "step": 153
+ },
+ {
+ "epoch": 1.37,
+ "grad_norm": 0.2262428264639853,
+ "learning_rate": 0.00015165093898182562,
+ "loss": 1.0066,
+ "step": 154
+ },
+ {
+ "epoch": 1.38,
+ "grad_norm": 0.22253433035020442,
+ "learning_rate": 0.00015103402886500525,
+ "loss": 0.8875,
+ "step": 155
+ },
+ {
+ "epoch": 1.39,
+ "grad_norm": 0.181161648904613,
+ "learning_rate": 0.00015041448123368455,
+ "loss": 0.9004,
+ "step": 156
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 0.20968483802367816,
+ "learning_rate": 0.00014979232810700637,
+ "loss": 0.9133,
+ "step": 157
+ },
+ {
+ "epoch": 1.41,
+ "grad_norm": 0.20540509271288435,
+ "learning_rate": 0.0001491676016387694,
+ "loss": 0.8876,
+ "step": 158
+ },
+ {
+ "epoch": 1.42,
+ "grad_norm": 0.18762795731312454,
+ "learning_rate": 0.00014854033411576659,
+ "loss": 0.933,
+ "step": 159
+ },
+ {
+ "epoch": 1.42,
+ "grad_norm": 0.23223345997338857,
+ "learning_rate": 0.00014791055795611624,
+ "loss": 0.9182,
+ "step": 160
+ },
+ {
+ "epoch": 1.43,
+ "grad_norm": 0.21932384461027146,
+ "learning_rate": 0.00014727830570758678,
+ "loss": 0.9514,
+ "step": 161
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.21819663730951108,
+ "learning_rate": 0.0001466436100459146,
+ "loss": 0.9162,
+ "step": 162
+ },
+ {
+ "epoch": 1.45,
+ "grad_norm": 0.2325813323476676,
+ "learning_rate": 0.00014600650377311522,
+ "loss": 0.9308,
+ "step": 163
+ },
+ {
+ "epoch": 1.46,
+ "grad_norm": 0.2568337182939043,
+ "learning_rate": 0.0001453670198157883,
+ "loss": 0.9995,
+ "step": 164
+ },
+ {
+ "epoch": 1.47,
+ "grad_norm": 0.22578454460723413,
+ "learning_rate": 0.00014472519122341566,
+ "loss": 0.9052,
+ "step": 165
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 0.23564258958796755,
+ "learning_rate": 0.00014408105116665336,
+ "loss": 0.9714,
+ "step": 166
+ },
+ {
+ "epoch": 1.49,
+ "grad_norm": 0.24266133562839415,
+ "learning_rate": 0.00014343463293561734,
+ "loss": 0.9219,
+ "step": 167
+ },
+ {
+ "epoch": 1.5,
+ "grad_norm": 0.23472454708184465,
+ "learning_rate": 0.00014278596993816308,
+ "loss": 0.8762,
+ "step": 168
+ },
+ {
+ "epoch": 1.5,
+ "eval_loss": 1.2197421789169312,
+ "eval_runtime": 13.2616,
+ "eval_samples_per_second": 22.622,
+ "eval_steps_per_second": 2.865,
+ "step": 168
+ },
+ {
+ "epoch": 1.5,
+ "grad_norm": 0.23623633375452713,
+ "learning_rate": 0.00014213509569815884,
+ "loss": 0.8809,
+ "step": 169
+ },
+ {
+ "epoch": 1.51,
+ "grad_norm": 0.25344275204523486,
+ "learning_rate": 0.00014148204385375321,
+ "loss": 0.7972,
+ "step": 170
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 0.23111396119549557,
+ "learning_rate": 0.0001408268481556366,
+ "loss": 0.8228,
+ "step": 171
+ },
+ {
+ "epoch": 1.53,
+ "grad_norm": 0.2510618369255398,
+ "learning_rate": 0.00014016954246529696,
+ "loss": 0.8849,
+ "step": 172
+ },
+ {
+ "epoch": 1.54,
+ "grad_norm": 0.2764366116622668,
+ "learning_rate": 0.0001395101607532698,
+ "loss": 0.8936,
+ "step": 173
+ },
+ {
+ "epoch": 1.55,
+ "grad_norm": 0.24325811719582827,
+ "learning_rate": 0.00013884873709738257,
+ "loss": 0.8602,
+ "step": 174
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 0.213781513838486,
+ "learning_rate": 0.00013818530568099327,
+ "loss": 0.9492,
+ "step": 175
+ },
+ {
+ "epoch": 1.57,
+ "grad_norm": 0.2397396374239057,
+ "learning_rate": 0.00013751990079122412,
+ "loss": 1.0499,
+ "step": 176
+ },
+ {
+ "epoch": 1.58,
+ "grad_norm": 0.21579907170368723,
+ "learning_rate": 0.00013685255681718922,
+ "loss": 0.9438,
+ "step": 177
+ },
+ {
+ "epoch": 1.58,
+ "grad_norm": 0.2359312681928786,
+ "learning_rate": 0.0001361833082482175,
+ "loss": 0.9289,
+ "step": 178
+ },
+ {
+ "epoch": 1.59,
+ "grad_norm": 0.2618189093396496,
+ "learning_rate": 0.0001355121896720703,
+ "loss": 0.981,
+ "step": 179
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.20876513773174135,
+ "learning_rate": 0.00013483923577315348,
+ "loss": 0.82,
+ "step": 180
+ },
+ {
+ "epoch": 1.61,
+ "grad_norm": 0.22162748553995645,
+ "learning_rate": 0.00013416448133072526,
+ "loss": 1.0131,
+ "step": 181
+ },
+ {
+ "epoch": 1.62,
+ "grad_norm": 0.20975549982451164,
+ "learning_rate": 0.00013348796121709862,
+ "loss": 0.8763,
+ "step": 182
+ },
+ {
+ "epoch": 1.63,
+ "grad_norm": 0.22840397707525473,
+ "learning_rate": 0.00013280971039583906,
+ "loss": 0.949,
+ "step": 183
+ },
+ {
+ "epoch": 1.64,
+ "grad_norm": 0.23384636230161737,
+ "learning_rate": 0.0001321297639199575,
+ "loss": 0.9567,
+ "step": 184
+ },
+ {
+ "epoch": 1.65,
+ "grad_norm": 0.22905979409902957,
+ "learning_rate": 0.000131448156930099,
+ "loss": 0.9153,
+ "step": 185
+ },
+ {
+ "epoch": 1.66,
+ "grad_norm": 0.27620894683694563,
+ "learning_rate": 0.0001307649246527263,
+ "loss": 0.8246,
+ "step": 186
+ },
+ {
+ "epoch": 1.66,
+ "grad_norm": 0.23004170633106227,
+ "learning_rate": 0.0001300801023982995,
+ "loss": 1.0181,
+ "step": 187
+ },
+ {
+ "epoch": 1.67,
+ "grad_norm": 0.2219849136264378,
+ "learning_rate": 0.00012939372555945112,
+ "loss": 0.9535,
+ "step": 188
+ },
+ {
+ "epoch": 1.68,
+ "grad_norm": 0.24458750452490116,
+ "learning_rate": 0.0001287058296091567,
+ "loss": 0.8968,
+ "step": 189
+ },
+ {
+ "epoch": 1.69,
+ "grad_norm": 0.2564337740159555,
+ "learning_rate": 0.00012801645009890195,
+ "loss": 0.7955,
+ "step": 190
+ },
+ {
+ "epoch": 1.7,
+ "grad_norm": 0.24100850371438767,
+ "learning_rate": 0.0001273256226568451,
+ "loss": 0.9235,
+ "step": 191
+ },
+ {
+ "epoch": 1.71,
+ "grad_norm": 0.24757089527873732,
+ "learning_rate": 0.00012663338298597563,
+ "loss": 1.007,
+ "step": 192
+ },
+ {
+ "epoch": 1.72,
+ "grad_norm": 0.24701038583742888,
+ "learning_rate": 0.00012593976686226904,
+ "loss": 0.9885,
+ "step": 193
+ },
+ {
+ "epoch": 1.73,
+ "grad_norm": 0.26373721125634964,
+ "learning_rate": 0.0001252448101328381,
+ "loss": 0.8785,
+ "step": 194
+ },
+ {
+ "epoch": 1.74,
+ "grad_norm": 0.2227761464470136,
+ "learning_rate": 0.00012454854871407994,
+ "loss": 0.8806,
+ "step": 195
+ },
+ {
+ "epoch": 1.74,
+ "grad_norm": 0.2283950634350429,
+ "learning_rate": 0.00012385101858982005,
+ "loss": 0.9053,
+ "step": 196
+ },
+ {
+ "epoch": 1.74,
+ "eval_loss": 1.2198154926300049,
+ "eval_runtime": 13.2208,
+ "eval_samples_per_second": 22.692,
+ "eval_steps_per_second": 2.874,
+ "step": 196
+ },
+ {
+ "epoch": 1.75,
+ "grad_norm": 0.23406423788354982,
+ "learning_rate": 0.00012315225580945252,
+ "loss": 0.9397,
+ "step": 197
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 0.23807045727443327,
+ "learning_rate": 0.0001224522964860769,
+ "loss": 0.9712,
+ "step": 198
+ },
+ {
+ "epoch": 1.77,
+ "grad_norm": 0.2463614808838948,
+ "learning_rate": 0.00012175117679463187,
+ "loss": 0.8558,
+ "step": 199
+ },
+ {
+ "epoch": 1.78,
+ "grad_norm": 0.24737417059302014,
+ "learning_rate": 0.00012104893297002567,
+ "loss": 0.9723,
+ "step": 200
+ },
+ {
+ "epoch": 1.79,
+ "grad_norm": 0.243750688050595,
+ "learning_rate": 0.0001203456013052634,
+ "loss": 0.964,
+ "step": 201
+ },
+ {
+ "epoch": 1.8,
+ "grad_norm": 0.24572059557106538,
+ "learning_rate": 0.00011964121814957137,
+ "loss": 0.9109,
+ "step": 202
+ },
+ {
+ "epoch": 1.81,
+ "grad_norm": 0.24044117903962453,
+ "learning_rate": 0.00011893581990651848,
+ "loss": 1.0019,
+ "step": 203
+ },
+ {
+ "epoch": 1.82,
+ "grad_norm": 0.2737568489071465,
+ "learning_rate": 0.00011822944303213486,
+ "loss": 0.8893,
+ "step": 204
+ },
+ {
+ "epoch": 1.82,
+ "grad_norm": 0.24122455882790084,
+ "learning_rate": 0.00011752212403302784,
+ "loss": 0.9162,
+ "step": 205
+ },
+ {
+ "epoch": 1.83,
+ "grad_norm": 0.28991871401626856,
+ "learning_rate": 0.00011681389946449504,
+ "loss": 0.8555,
+ "step": 206
+ },
+ {
+ "epoch": 1.84,
+ "grad_norm": 0.23767408810646548,
+ "learning_rate": 0.00011610480592863531,
+ "loss": 0.9936,
+ "step": 207
+ },
+ {
+ "epoch": 1.85,
+ "grad_norm": 0.22614733706173062,
+ "learning_rate": 0.00011539488007245702,
+ "loss": 0.916,
+ "step": 208
+ },
+ {
+ "epoch": 1.86,
+ "grad_norm": 0.22471992425846515,
+ "learning_rate": 0.00011468415858598411,
+ "loss": 0.8872,
+ "step": 209
+ },
+ {
+ "epoch": 1.87,
+ "grad_norm": 0.22675717145909688,
+ "learning_rate": 0.00011397267820035986,
+ "loss": 0.8393,
+ "step": 210
+ },
+ {
+ "epoch": 1.88,
+ "grad_norm": 0.2727459336483823,
+ "learning_rate": 0.00011326047568594851,
+ "loss": 0.8265,
+ "step": 211
+ },
+ {
+ "epoch": 1.89,
+ "grad_norm": 0.25216778031670767,
+ "learning_rate": 0.00011254758785043515,
+ "loss": 0.9939,
+ "step": 212
+ },
+ {
+ "epoch": 1.9,
+ "grad_norm": 0.269147378424304,
+ "learning_rate": 0.0001118340515369232,
+ "loss": 0.9102,
+ "step": 213
+ },
+ {
+ "epoch": 1.91,
+ "grad_norm": 0.2216178370833471,
+ "learning_rate": 0.00011111990362203033,
+ "loss": 0.8778,
+ "step": 214
+ },
+ {
+ "epoch": 1.91,
+ "grad_norm": 0.2602474934716497,
+ "learning_rate": 0.00011040518101398276,
+ "loss": 0.9454,
+ "step": 215
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 0.2658635078442998,
+ "learning_rate": 0.00010968992065070769,
+ "loss": 0.8098,
+ "step": 216
+ },
+ {
+ "epoch": 1.93,
+ "grad_norm": 0.20997905209488962,
+ "learning_rate": 0.00010897415949792427,
+ "loss": 0.9318,
+ "step": 217
+ },
+ {
+ "epoch": 1.94,
+ "grad_norm": 0.24752453752221557,
+ "learning_rate": 0.00010825793454723325,
+ "loss": 0.949,
+ "step": 218
+ },
+ {
+ "epoch": 1.95,
+ "grad_norm": 0.255579569750529,
+ "learning_rate": 0.0001075412828142051,
+ "loss": 0.915,
+ "step": 219
+ },
+ {
+ "epoch": 1.96,
+ "grad_norm": 0.23186981930561867,
+ "learning_rate": 0.0001068242413364671,
+ "loss": 0.9132,
+ "step": 220
+ },
+ {
+ "epoch": 1.97,
+ "grad_norm": 0.35685140391438824,
+ "learning_rate": 0.00010610684717178905,
+ "loss": 0.9398,
+ "step": 221
+ },
+ {
+ "epoch": 1.98,
+ "grad_norm": 0.27320389987223703,
+ "learning_rate": 0.00010538913739616816,
+ "loss": 0.857,
+ "step": 222
+ },
+ {
+ "epoch": 1.99,
+ "grad_norm": 0.2324276771141761,
+ "learning_rate": 0.00010467114910191289,
+ "loss": 0.8546,
+ "step": 223
+ },
+ {
+ "epoch": 1.99,
+ "grad_norm": 0.22820341349854167,
+ "learning_rate": 0.00010395291939572593,
+ "loss": 0.9301,
+ "step": 224
+ },
+ {
+ "epoch": 1.99,
+ "eval_loss": 1.2246263027191162,
+ "eval_runtime": 13.1981,
+ "eval_samples_per_second": 22.731,
+ "eval_steps_per_second": 2.879,
+ "step": 224
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 0.2289800489154315,
+ "learning_rate": 0.00010323448539678653,
+ "loss": 0.9922,
+ "step": 225
+ },
+ {
+ "epoch": 2.01,
+ "grad_norm": 0.2673353778680862,
+ "learning_rate": 0.00010251588423483205,
+ "loss": 0.7779,
+ "step": 226
+ },
+ {
+ "epoch": 2.02,
+ "grad_norm": 0.2420933678952559,
+ "learning_rate": 0.0001017971530482392,
+ "loss": 0.8044,
+ "step": 227
+ },
+ {
+ "epoch": 2.03,
+ "grad_norm": 0.21799264660625498,
+ "learning_rate": 0.00010107832898210439,
+ "loss": 0.8773,
+ "step": 228
+ },
+ {
+ "epoch": 2.04,
+ "grad_norm": 0.21443255695871016,
+ "learning_rate": 0.00010035944918632429,
+ "loss": 0.9031,
+ "step": 229
+ },
+ {
+ "epoch": 2.05,
+ "grad_norm": 0.23983734165788242,
+ "learning_rate": 9.96405508136757e-05,
+ "loss": 0.9014,
+ "step": 230
+ },
+ {
+ "epoch": 2.06,
+ "grad_norm": 0.27915481475799336,
+ "learning_rate": 9.892167101789564e-05,
+ "loss": 0.8853,
+ "step": 231
+ },
+ {
+ "epoch": 2.07,
+ "grad_norm": 0.2688949371564916,
+ "learning_rate": 9.820284695176082e-05,
+ "loss": 0.8452,
+ "step": 232
+ },
+ {
+ "epoch": 2.07,
+ "grad_norm": 0.2623278518867105,
+ "learning_rate": 9.748411576516794e-05,
+ "loss": 0.8612,
+ "step": 233
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 0.2710502639103885,
+ "learning_rate": 9.676551460321349e-05,
+ "loss": 0.8108,
+ "step": 234
+ },
+ {
+ "epoch": 2.09,
+ "grad_norm": 0.282572880285737,
+ "learning_rate": 9.60470806042741e-05,
+ "loss": 0.7866,
+ "step": 235
+ },
+ {
+ "epoch": 2.1,
+ "grad_norm": 0.2829396962922612,
+ "learning_rate": 9.532885089808713e-05,
+ "loss": 0.8557,
+ "step": 236
+ },
+ {
+ "epoch": 2.11,
+ "grad_norm": 0.2721172338857335,
+ "learning_rate": 9.461086260383187e-05,
+ "loss": 0.7933,
+ "step": 237
+ },
+ {
+ "epoch": 2.12,
+ "grad_norm": 0.29736638811364446,
+ "learning_rate": 9.389315282821097e-05,
+ "loss": 0.7674,
+ "step": 238
+ },
+ {
+ "epoch": 2.13,
+ "grad_norm": 0.28571679920981263,
+ "learning_rate": 9.317575866353292e-05,
+ "loss": 0.7442,
+ "step": 239
+ },
+ {
+ "epoch": 2.14,
+ "grad_norm": 0.264545167150173,
+ "learning_rate": 9.245871718579491e-05,
+ "loss": 0.8505,
+ "step": 240
+ },
+ {
+ "epoch": 2.15,
+ "grad_norm": 0.30691085134027757,
+ "learning_rate": 9.174206545276677e-05,
+ "loss": 0.7898,
+ "step": 241
+ },
+ {
+ "epoch": 2.15,
+ "grad_norm": 0.31375028121981235,
+ "learning_rate": 9.102584050207578e-05,
+ "loss": 0.7661,
+ "step": 242
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 0.28421530221837016,
+ "learning_rate": 9.031007934929236e-05,
+ "loss": 0.8328,
+ "step": 243
+ },
+ {
+ "epoch": 2.17,
+ "grad_norm": 0.25601367811173414,
+ "learning_rate": 8.959481898601728e-05,
+ "loss": 0.8281,
+ "step": 244
+ },
+ {
+ "epoch": 2.18,
+ "grad_norm": 0.2983724947729522,
+ "learning_rate": 8.888009637796968e-05,
+ "loss": 0.8567,
+ "step": 245
+ },
+ {
+ "epoch": 2.19,
+ "grad_norm": 0.2545616786933236,
+ "learning_rate": 8.81659484630768e-05,
+ "loss": 0.9151,
+ "step": 246
+ },
+ {
+ "epoch": 2.2,
+ "grad_norm": 0.23873712362647942,
+ "learning_rate": 8.745241214956483e-05,
+ "loss": 0.8818,
+ "step": 247
+ },
+ {
+ "epoch": 2.21,
+ "grad_norm": 0.285331972404065,
+ "learning_rate": 8.673952431405148e-05,
+ "loss": 0.7983,
+ "step": 248
+ },
+ {
+ "epoch": 2.22,
+ "grad_norm": 0.23897707291689843,
+ "learning_rate": 8.602732179964017e-05,
+ "loss": 0.8758,
+ "step": 249
+ },
+ {
+ "epoch": 2.23,
+ "grad_norm": 0.2830966091447457,
+ "learning_rate": 8.531584141401591e-05,
+ "loss": 0.8714,
+ "step": 250
+ },
+ {
+ "epoch": 2.23,
+ "grad_norm": 0.28872599217076506,
+ "learning_rate": 8.4605119927543e-05,
+ "loss": 0.8387,
+ "step": 251
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 0.2652236346400331,
+ "learning_rate": 8.38951940713647e-05,
+ "loss": 0.8232,
+ "step": 252
+ },
+ {
+ "epoch": 2.24,
+ "eval_loss": 1.2432794570922852,
+ "eval_runtime": 13.2405,
+ "eval_samples_per_second": 22.658,
+ "eval_steps_per_second": 2.87,
+ "step": 252
+ },
+ {
+ "epoch": 2.25,
+ "grad_norm": 0.299978013524394,
+ "learning_rate": 8.318610053550497e-05,
+ "loss": 0.7321,
+ "step": 253
+ },
+ {
+ "epoch": 2.26,
+ "grad_norm": 0.2740002835117391,
+ "learning_rate": 8.247787596697218e-05,
+ "loss": 0.7605,
+ "step": 254
+ },
+ {
+ "epoch": 2.27,
+ "grad_norm": 0.2848366030132808,
+ "learning_rate": 8.177055696786516e-05,
+ "loss": 0.8485,
+ "step": 255
+ },
+ {
+ "epoch": 2.28,
+ "grad_norm": 0.24847418856075218,
+ "learning_rate": 8.106418009348157e-05,
+ "loss": 0.7557,
+ "step": 256
+ },
+ {
+ "epoch": 2.29,
+ "grad_norm": 0.33515508602624905,
+ "learning_rate": 8.035878185042868e-05,
+ "loss": 0.8015,
+ "step": 257
+ },
+ {
+ "epoch": 2.3,
+ "grad_norm": 0.2905943721096322,
+ "learning_rate": 7.965439869473664e-05,
+ "loss": 0.8457,
+ "step": 258
+ },
+ {
+ "epoch": 2.31,
+ "grad_norm": 0.3140679719552616,
+ "learning_rate": 7.895106702997437e-05,
+ "loss": 0.8559,
+ "step": 259
+ },
+ {
+ "epoch": 2.31,
+ "grad_norm": 0.29745105018138573,
+ "learning_rate": 7.824882320536814e-05,
+ "loss": 0.7453,
+ "step": 260
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 0.29818631731197365,
+ "learning_rate": 7.754770351392311e-05,
+ "loss": 0.8354,
+ "step": 261
+ },
+ {
+ "epoch": 2.33,
+ "grad_norm": 0.24721488944366407,
+ "learning_rate": 7.684774419054747e-05,
+ "loss": 0.7755,
+ "step": 262
+ },
+ {
+ "epoch": 2.34,
+ "grad_norm": 0.31210442779019465,
+ "learning_rate": 7.614898141017996e-05,
+ "loss": 0.7208,
+ "step": 263
+ },
+ {
+ "epoch": 2.35,
+ "grad_norm": 0.2873220240109992,
+ "learning_rate": 7.54514512859201e-05,
+ "loss": 0.7548,
+ "step": 264
+ },
+ {
+ "epoch": 2.36,
+ "grad_norm": 0.3006634171776217,
+ "learning_rate": 7.475518986716194e-05,
+ "loss": 0.7566,
+ "step": 265
+ },
+ {
+ "epoch": 2.37,
+ "grad_norm": 0.2799417613336026,
+ "learning_rate": 7.406023313773097e-05,
+ "loss": 0.727,
+ "step": 266
+ },
+ {
+ "epoch": 2.38,
+ "grad_norm": 0.2451761866231664,
+ "learning_rate": 7.336661701402439e-05,
+ "loss": 0.9641,
+ "step": 267
+ },
+ {
+ "epoch": 2.39,
+ "grad_norm": 0.305202611125298,
+ "learning_rate": 7.267437734315492e-05,
+ "loss": 0.7891,
+ "step": 268
+ },
+ {
+ "epoch": 2.39,
+ "grad_norm": 0.29107717848747816,
+ "learning_rate": 7.198354990109805e-05,
+ "loss": 0.9032,
+ "step": 269
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 0.2688898665176787,
+ "learning_rate": 7.129417039084333e-05,
+ "loss": 0.8416,
+ "step": 270
+ },
+ {
+ "epoch": 2.41,
+ "grad_norm": 0.2814206029778395,
+ "learning_rate": 7.060627444054893e-05,
+ "loss": 0.8443,
+ "step": 271
+ },
+ {
+ "epoch": 2.42,
+ "grad_norm": 0.2862094867555512,
+ "learning_rate": 6.99198976017005e-05,
+ "loss": 0.8271,
+ "step": 272
+ },
+ {
+ "epoch": 2.43,
+ "grad_norm": 0.3214647340394826,
+ "learning_rate": 6.923507534727373e-05,
+ "loss": 0.7793,
+ "step": 273
+ },
+ {
+ "epoch": 2.44,
+ "grad_norm": 0.3033659714564417,
+ "learning_rate": 6.855184306990106e-05,
+ "loss": 0.7856,
+ "step": 274
+ },
+ {
+ "epoch": 2.45,
+ "grad_norm": 0.3024382342577774,
+ "learning_rate": 6.78702360800425e-05,
+ "loss": 0.8633,
+ "step": 275
+ },
+ {
+ "epoch": 2.46,
+ "grad_norm": 0.25803598196729505,
+ "learning_rate": 6.719028960416098e-05,
+ "loss": 0.8428,
+ "step": 276
+ },
+ {
+ "epoch": 2.47,
+ "grad_norm": 0.35469202971401803,
+ "learning_rate": 6.651203878290139e-05,
+ "loss": 0.8665,
+ "step": 277
+ },
+ {
+ "epoch": 2.47,
+ "grad_norm": 0.3122516837597691,
+ "learning_rate": 6.583551866927475e-05,
+ "loss": 0.8787,
+ "step": 278
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 0.3305470786367901,
+ "learning_rate": 6.516076422684654e-05,
+ "loss": 0.8765,
+ "step": 279
+ },
+ {
+ "epoch": 2.49,
+ "grad_norm": 0.3324622666488467,
+ "learning_rate": 6.448781032792972e-05,
+ "loss": 0.8318,
+ "step": 280
+ },
+ {
+ "epoch": 2.49,
+ "eval_loss": 1.2546111345291138,
+ "eval_runtime": 13.2379,
+ "eval_samples_per_second": 22.662,
+ "eval_steps_per_second": 2.871,
+ "step": 280
+ },
+ {
+ "epoch": 2.5,
+ "grad_norm": 0.342341713579355,
+ "learning_rate": 6.381669175178248e-05,
+ "loss": 0.9517,
+ "step": 281
+ },
+ {
+ "epoch": 2.51,
+ "grad_norm": 0.33913458352374665,
+ "learning_rate": 6.31474431828108e-05,
+ "loss": 0.8564,
+ "step": 282
+ },
+ {
+ "epoch": 2.52,
+ "grad_norm": 0.30528689383480295,
+ "learning_rate": 6.248009920877592e-05,
+ "loss": 0.8199,
+ "step": 283
+ },
+ {
+ "epoch": 2.53,
+ "grad_norm": 0.29698648367254743,
+ "learning_rate": 6.181469431900672e-05,
+ "loss": 0.785,
+ "step": 284
+ },
+ {
+ "epoch": 2.54,
+ "grad_norm": 0.32239262939282626,
+ "learning_rate": 6.115126290261745e-05,
+ "loss": 0.7794,
+ "step": 285
+ },
+ {
+ "epoch": 2.55,
+ "grad_norm": 0.2694595905080167,
+ "learning_rate": 6.048983924673022e-05,
+ "loss": 0.8056,
+ "step": 286
+ },
+ {
+ "epoch": 2.55,
+ "grad_norm": 0.3045496751154443,
+ "learning_rate": 5.983045753470308e-05,
+ "loss": 0.8164,
+ "step": 287
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 0.2927868214627918,
+ "learning_rate": 5.917315184436345e-05,
+ "loss": 0.8358,
+ "step": 288
+ },
+ {
+ "epoch": 2.57,
+ "grad_norm": 0.2931914055644858,
+ "learning_rate": 5.851795614624682e-05,
+ "loss": 0.8011,
+ "step": 289
+ },
+ {
+ "epoch": 2.58,
+ "grad_norm": 0.3158716819379082,
+ "learning_rate": 5.786490430184115e-05,
+ "loss": 0.8332,
+ "step": 290
+ },
+ {
+ "epoch": 2.59,
+ "grad_norm": 0.3482519147352008,
+ "learning_rate": 5.72140300618369e-05,
+ "loss": 0.7621,
+ "step": 291
+ },
+ {
+ "epoch": 2.6,
+ "grad_norm": 0.28652801822050894,
+ "learning_rate": 5.656536706438267e-05,
+ "loss": 0.77,
+ "step": 292
+ },
+ {
+ "epoch": 2.61,
+ "grad_norm": 0.29691290613407717,
+ "learning_rate": 5.591894883334667e-05,
+ "loss": 0.9394,
+ "step": 293
+ },
+ {
+ "epoch": 2.62,
+ "grad_norm": 0.26699581966985203,
+ "learning_rate": 5.5274808776584367e-05,
+ "loss": 0.7918,
+ "step": 294
+ },
+ {
+ "epoch": 2.63,
+ "grad_norm": 0.2926923719762685,
+ "learning_rate": 5.463298018421171e-05,
+ "loss": 0.8723,
+ "step": 295
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.3403087263187063,
+ "learning_rate": 5.399349622688479e-05,
+ "loss": 0.8097,
+ "step": 296
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.34261233464532476,
+ "learning_rate": 5.335638995408545e-05,
+ "loss": 0.9032,
+ "step": 297
+ },
+ {
+ "epoch": 2.65,
+ "grad_norm": 0.31315234759634086,
+ "learning_rate": 5.272169429241325e-05,
+ "loss": 0.82,
+ "step": 298
+ },
+ {
+ "epoch": 2.66,
+ "grad_norm": 0.3179759425444047,
+ "learning_rate": 5.208944204388377e-05,
+ "loss": 0.8864,
+ "step": 299
+ },
+ {
+ "epoch": 2.67,
+ "grad_norm": 0.3121296356843828,
+ "learning_rate": 5.145966588423341e-05,
+ "loss": 0.8258,
+ "step": 300
+ },
+ {
+ "epoch": 2.68,
+ "grad_norm": 0.268436849924173,
+ "learning_rate": 5.0832398361230596e-05,
+ "loss": 0.8906,
+ "step": 301
+ },
+ {
+ "epoch": 2.69,
+ "grad_norm": 0.2961161602467319,
+ "learning_rate": 5.020767189299369e-05,
+ "loss": 0.8828,
+ "step": 302
+ },
+ {
+ "epoch": 2.7,
+ "grad_norm": 0.27743957099992345,
+ "learning_rate": 4.9585518766315496e-05,
+ "loss": 0.8251,
+ "step": 303
+ },
+ {
+ "epoch": 2.71,
+ "grad_norm": 0.2949909861852426,
+ "learning_rate": 4.896597113499479e-05,
+ "loss": 0.7911,
+ "step": 304
+ },
+ {
+ "epoch": 2.72,
+ "grad_norm": 0.3161115451278363,
+ "learning_rate": 4.834906101817438e-05,
+ "loss": 0.8157,
+ "step": 305
+ },
+ {
+ "epoch": 2.72,
+ "grad_norm": 0.28720077046065867,
+ "learning_rate": 4.773482029868657e-05,
+ "loss": 0.82,
+ "step": 306
+ },
+ {
+ "epoch": 2.73,
+ "grad_norm": 0.4045319357608716,
+ "learning_rate": 4.712328072140505e-05,
+ "loss": 0.8414,
+ "step": 307
+ },
+ {
+ "epoch": 2.74,
+ "grad_norm": 0.3070232288390269,
+ "learning_rate": 4.651447389160458e-05,
+ "loss": 0.8427,
+ "step": 308
+ },
+ {
+ "epoch": 2.74,
+ "eval_loss": 1.2574400901794434,
+ "eval_runtime": 13.2473,
+ "eval_samples_per_second": 22.646,
+ "eval_steps_per_second": 2.869,
+ "step": 308
+ },
+ {
+ "epoch": 2.75,
+ "grad_norm": 0.3214782806968351,
+ "learning_rate": 4.5908431273327436e-05,
+ "loss": 0.8469,
+ "step": 309
+ },
+ {
+ "epoch": 2.76,
+ "grad_norm": 0.24241410698156174,
+ "learning_rate": 4.530518418775733e-05,
+ "loss": 0.8346,
+ "step": 310
+ },
+ {
+ "epoch": 2.77,
+ "grad_norm": 0.3303263594210879,
+ "learning_rate": 4.470476381160065e-05,
+ "loss": 0.8298,
+ "step": 311
+ },
+ {
+ "epoch": 2.78,
+ "grad_norm": 0.30711900849760865,
+ "learning_rate": 4.4107201175475275e-05,
+ "loss": 0.789,
+ "step": 312
+ },
+ {
+ "epoch": 2.79,
+ "grad_norm": 0.2954465859389713,
+ "learning_rate": 4.351252716230685e-05,
+ "loss": 0.8029,
+ "step": 313
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 0.29925087091531116,
+ "learning_rate": 4.292077250573266e-05,
+ "loss": 0.8633,
+ "step": 314
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 0.3177611223775825,
+ "learning_rate": 4.2331967788513295e-05,
+ "loss": 0.76,
+ "step": 315
+ },
+ {
+ "epoch": 2.81,
+ "grad_norm": 0.28642407848269513,
+ "learning_rate": 4.174614344095213e-05,
+ "loss": 0.823,
+ "step": 316
+ },
+ {
+ "epoch": 2.82,
+ "grad_norm": 0.3243224656005062,
+ "learning_rate": 4.116332973932256e-05,
+ "loss": 0.7831,
+ "step": 317
+ },
+ {
+ "epoch": 2.83,
+ "grad_norm": 0.34877334027822726,
+ "learning_rate": 4.058355680430337e-05,
+ "loss": 0.899,
+ "step": 318
+ },
+ {
+ "epoch": 2.84,
+ "grad_norm": 0.28640325479143114,
+ "learning_rate": 4.0006854599421926e-05,
+ "loss": 0.8292,
+ "step": 319
+ },
+ {
+ "epoch": 2.85,
+ "grad_norm": 0.3135316628014535,
+ "learning_rate": 3.943325292950579e-05,
+ "loss": 0.8731,
+ "step": 320
+ },
+ {
+ "epoch": 2.86,
+ "grad_norm": 0.2949970604257085,
+ "learning_rate": 3.886278143914219e-05,
+ "loss": 0.8402,
+ "step": 321
+ },
+ {
+ "epoch": 2.87,
+ "grad_norm": 0.30057896586780075,
+ "learning_rate": 3.829546961114607e-05,
+ "loss": 0.7713,
+ "step": 322
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 0.3558574270285126,
+ "learning_rate": 3.773134676503629e-05,
+ "loss": 0.8435,
+ "step": 323
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 0.29115288332943334,
+ "learning_rate": 3.7170442055520415e-05,
+ "loss": 0.9022,
+ "step": 324
+ },
+ {
+ "epoch": 2.89,
+ "grad_norm": 0.3192074718527619,
+ "learning_rate": 3.661278447098789e-05,
+ "loss": 0.7662,
+ "step": 325
+ },
+ {
+ "epoch": 2.9,
+ "grad_norm": 0.33335742888185405,
+ "learning_rate": 3.605840283201195e-05,
+ "loss": 0.8111,
+ "step": 326
+ },
+ {
+ "epoch": 2.91,
+ "grad_norm": 0.29748212071395186,
+ "learning_rate": 3.550732578986006e-05,
+ "loss": 0.7543,
+ "step": 327
+ },
+ {
+ "epoch": 2.92,
+ "grad_norm": 0.3680409192627914,
+ "learning_rate": 3.495958182501325e-05,
+ "loss": 0.8124,
+ "step": 328
+ },
+ {
+ "epoch": 2.93,
+ "grad_norm": 0.27807302364345643,
+ "learning_rate": 3.441519924569408e-05,
+ "loss": 0.7856,
+ "step": 329
+ },
+ {
+ "epoch": 2.94,
+ "grad_norm": 0.3050855823733691,
+ "learning_rate": 3.387420618640379e-05,
+ "loss": 0.8506,
+ "step": 330
+ },
+ {
+ "epoch": 2.95,
+ "grad_norm": 0.3322620263029238,
+ "learning_rate": 3.3336630606468134e-05,
+ "loss": 0.8771,
+ "step": 331
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 0.3112008867427982,
+ "learning_rate": 3.280250028859248e-05,
+ "loss": 0.7785,
+ "step": 332
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 0.2839548329095365,
+ "learning_rate": 3.227184283742591e-05,
+ "loss": 0.9153,
+ "step": 333
+ },
+ {
+ "epoch": 2.97,
+ "grad_norm": 0.34615397822650606,
+ "learning_rate": 3.174468567813461e-05,
+ "loss": 0.7753,
+ "step": 334
+ },
+ {
+ "epoch": 2.98,
+ "grad_norm": 0.34691866307772695,
+ "learning_rate": 3.122105605498442e-05,
+ "loss": 0.851,
+ "step": 335
+ },
+ {
+ "epoch": 2.99,
+ "grad_norm": 0.296369624391198,
+ "learning_rate": 3.070098102993302e-05,
+ "loss": 0.8572,
+ "step": 336
+ },
+ {
+ "epoch": 2.99,
+ "eval_loss": 1.2511259317398071,
+ "eval_runtime": 13.2202,
+ "eval_samples_per_second": 22.692,
+ "eval_steps_per_second": 2.874,
+ "step": 336
+ },
+ {
+ "epoch": 3.0,
+ "grad_norm": 0.2960426809914107,
+ "learning_rate": 3.018448748123097e-05,
+ "loss": 0.8664,
+ "step": 337
+ },
+ {
+ "epoch": 3.01,
+ "grad_norm": 0.3213684656412904,
+ "learning_rate": 2.9671602102032926e-05,
+ "loss": 0.7387,
+ "step": 338
+ },
+ {
+ "epoch": 3.02,
+ "grad_norm": 0.3057515564517862,
+ "learning_rate": 2.9162351399017963e-05,
+ "loss": 0.767,
+ "step": 339
+ },
+ {
+ "epoch": 3.03,
+ "grad_norm": 0.31163087624473346,
+ "learning_rate": 2.8656761691019673e-05,
+ "loss": 0.8084,
+ "step": 340
+ },
+ {
+ "epoch": 3.04,
+ "grad_norm": 0.29945884165692627,
+ "learning_rate": 2.8154859107665987e-05,
+ "loss": 0.9033,
+ "step": 341
+ },
+ {
+ "epoch": 3.04,
+ "grad_norm": 0.32235895597138414,
+ "learning_rate": 2.7656669588028762e-05,
+ "loss": 0.7144,
+ "step": 342
+ },
+ {
+ "epoch": 3.05,
+ "grad_norm": 0.3458958643468077,
+ "learning_rate": 2.7162218879283176e-05,
+ "loss": 0.7426,
+ "step": 343
+ },
+ {
+ "epoch": 3.06,
+ "grad_norm": 0.3073811670129139,
+ "learning_rate": 2.667153253537713e-05,
+ "loss": 0.723,
+ "step": 344
+ },
+ {
+ "epoch": 3.07,
+ "grad_norm": 0.3223615792503137,
+ "learning_rate": 2.618463591571052e-05,
+ "loss": 0.6815,
+ "step": 345
+ },
+ {
+ "epoch": 3.08,
+ "grad_norm": 0.357176712253696,
+ "learning_rate": 2.570155418382473e-05,
+ "loss": 0.7286,
+ "step": 346
+ },
+ {
+ "epoch": 3.09,
+ "grad_norm": 0.29547503050650586,
+ "learning_rate": 2.5222312306101925e-05,
+ "loss": 0.7895,
+ "step": 347
+ },
+ {
+ "epoch": 3.1,
+ "grad_norm": 0.37115835330190844,
+ "learning_rate": 2.474693505047504e-05,
+ "loss": 0.8722,
+ "step": 348
+ },
+ {
+ "epoch": 3.11,
+ "grad_norm": 0.3840299049618982,
+ "learning_rate": 2.427544698514753e-05,
+ "loss": 0.7022,
+ "step": 349
+ },
+ {
+ "epoch": 3.12,
+ "grad_norm": 0.33738326420366743,
+ "learning_rate": 2.3807872477323733e-05,
+ "loss": 0.7731,
+ "step": 350
+ },
+ {
+ "epoch": 3.12,
+ "grad_norm": 0.35407291134427865,
+ "learning_rate": 2.334423569194948e-05,
+ "loss": 0.7407,
+ "step": 351
+ },
+ {
+ "epoch": 3.13,
+ "grad_norm": 0.2589965276614251,
+ "learning_rate": 2.288456059046331e-05,
+ "loss": 0.865,
+ "step": 352
+ },
+ {
+ "epoch": 3.14,
+ "grad_norm": 0.3521332875577516,
+ "learning_rate": 2.242887092955801e-05,
+ "loss": 0.655,
+ "step": 353
+ },
+ {
+ "epoch": 3.15,
+ "grad_norm": 0.33409347759889046,
+ "learning_rate": 2.1977190259952883e-05,
+ "loss": 0.7536,
+ "step": 354
+ },
+ {
+ "epoch": 3.16,
+ "grad_norm": 0.38883294070164465,
+ "learning_rate": 2.1529541925176555e-05,
+ "loss": 0.7435,
+ "step": 355
+ },
+ {
+ "epoch": 3.17,
+ "grad_norm": 0.3513324514694806,
+ "learning_rate": 2.1085949060360654e-05,
+ "loss": 0.7405,
+ "step": 356
+ },
+ {
+ "epoch": 3.18,
+ "grad_norm": 0.3391182488874308,
+ "learning_rate": 2.064643459104405e-05,
+ "loss": 0.7737,
+ "step": 357
+ },
+ {
+ "epoch": 3.19,
+ "grad_norm": 0.3647814711811327,
+ "learning_rate": 2.0211021231988102e-05,
+ "loss": 0.7095,
+ "step": 358
+ },
+ {
+ "epoch": 3.2,
+ "grad_norm": 0.35539668264402535,
+ "learning_rate": 1.977973148600266e-05,
+ "loss": 0.7543,
+ "step": 359
+ },
+ {
+ "epoch": 3.2,
+ "grad_norm": 0.30454971181182305,
+ "learning_rate": 1.935258764278314e-05,
+ "loss": 0.7897,
+ "step": 360
+ },
+ {
+ "epoch": 3.21,
+ "grad_norm": 0.33290039075033256,
+ "learning_rate": 1.8929611777758526e-05,
+ "loss": 0.6976,
+ "step": 361
+ },
+ {
+ "epoch": 3.22,
+ "grad_norm": 0.28252566634530724,
+ "learning_rate": 1.851082575095051e-05,
+ "loss": 0.8688,
+ "step": 362
+ },
+ {
+ "epoch": 3.23,
+ "grad_norm": 0.38585709417657277,
+ "learning_rate": 1.8096251205843684e-05,
+ "loss": 0.8361,
+ "step": 363
+ },
+ {
+ "epoch": 3.24,
+ "grad_norm": 0.3372165483048113,
+ "learning_rate": 1.7685909568267033e-05,
+ "loss": 0.7818,
+ "step": 364
+ },
+ {
+ "epoch": 3.24,
+ "eval_loss": 1.2727638483047485,
+ "eval_runtime": 13.2462,
+ "eval_samples_per_second": 22.648,
+ "eval_steps_per_second": 2.869,
+ "step": 364
+ },
+ {
+ "epoch": 3.25,
+ "grad_norm": 0.3250536223337791,
+ "learning_rate": 1.7279822045286576e-05,
+ "loss": 0.7973,
+ "step": 365
+ },
+ {
+ "epoch": 3.26,
+ "grad_norm": 0.3593434237000891,
+ "learning_rate": 1.6878009624109313e-05,
+ "loss": 0.7015,
+ "step": 366
+ },
+ {
+ "epoch": 3.27,
+ "grad_norm": 0.3197427893489943,
+ "learning_rate": 1.648049307099874e-05,
+ "loss": 0.7536,
+ "step": 367
+ },
+ {
+ "epoch": 3.28,
+ "grad_norm": 0.36529609852202527,
+ "learning_rate": 1.6087292930201394e-05,
+ "loss": 0.7487,
+ "step": 368
+ },
+ {
+ "epoch": 3.28,
+ "grad_norm": 0.25007649085986594,
+ "learning_rate": 1.569842952288527e-05,
+ "loss": 0.8906,
+ "step": 369
+ },
+ {
+ "epoch": 3.29,
+ "grad_norm": 0.36329388340138336,
+ "learning_rate": 1.5313922946089486e-05,
+ "loss": 0.7645,
+ "step": 370
+ },
+ {
+ "epoch": 3.3,
+ "grad_norm": 0.35241947629057724,
+ "learning_rate": 1.4933793071685732e-05,
+ "loss": 0.8134,
+ "step": 371
+ },
+ {
+ "epoch": 3.31,
+ "grad_norm": 0.3664249862728154,
+ "learning_rate": 1.4558059545351143e-05,
+ "loss": 0.7328,
+ "step": 372
+ },
+ {
+ "epoch": 3.32,
+ "grad_norm": 0.29196433505411595,
+ "learning_rate": 1.4186741785553115e-05,
+ "loss": 0.7608,
+ "step": 373
+ },
+ {
+ "epoch": 3.33,
+ "grad_norm": 0.2923688341161706,
+ "learning_rate": 1.3819858982545598e-05,
+ "loss": 0.7696,
+ "step": 374
+ },
+ {
+ "epoch": 3.34,
+ "grad_norm": 0.3520285590111714,
+ "learning_rate": 1.3457430097377421e-05,
+ "loss": 0.7871,
+ "step": 375
+ },
+ {
+ "epoch": 3.35,
+ "grad_norm": 0.30838315755571466,
+ "learning_rate": 1.3099473860912326e-05,
+ "loss": 0.7398,
+ "step": 376
+ },
+ {
+ "epoch": 3.36,
+ "grad_norm": 0.327647793123604,
+ "learning_rate": 1.2746008772860884e-05,
+ "loss": 0.7974,
+ "step": 377
+ },
+ {
+ "epoch": 3.36,
+ "grad_norm": 0.29911511172677985,
+ "learning_rate": 1.2397053100824463e-05,
+ "loss": 0.8535,
+ "step": 378
+ },
+ {
+ "epoch": 3.37,
+ "grad_norm": 0.34600894138601895,
+ "learning_rate": 1.2052624879351104e-05,
+ "loss": 0.6872,
+ "step": 379
+ },
+ {
+ "epoch": 3.38,
+ "grad_norm": 0.2993988371658933,
+ "learning_rate": 1.1712741909003444e-05,
+ "loss": 0.8554,
+ "step": 380
+ },
+ {
+ "epoch": 3.39,
+ "grad_norm": 0.34130407502289717,
+ "learning_rate": 1.1377421755438832e-05,
+ "loss": 0.9014,
+ "step": 381
+ },
+ {
+ "epoch": 3.4,
+ "grad_norm": 0.33654746414289566,
+ "learning_rate": 1.1046681748501408e-05,
+ "loss": 0.853,
+ "step": 382
+ },
+ {
+ "epoch": 3.41,
+ "grad_norm": 0.32172878518370435,
+ "learning_rate": 1.0720538981326556e-05,
+ "loss": 0.7594,
+ "step": 383
+ },
+ {
+ "epoch": 3.42,
+ "grad_norm": 0.3211202670840339,
+ "learning_rate": 1.0399010309457457e-05,
+ "loss": 0.8017,
+ "step": 384
+ },
+ {
+ "epoch": 3.43,
+ "grad_norm": 0.31506066245322806,
+ "learning_rate": 1.0082112349974016e-05,
+ "loss": 0.7279,
+ "step": 385
+ },
+ {
+ "epoch": 3.44,
+ "grad_norm": 0.33655604261545546,
+ "learning_rate": 9.76986148063398e-06,
+ "loss": 0.8117,
+ "step": 386
+ },
+ {
+ "epoch": 3.45,
+ "grad_norm": 0.34046018532794836,
+ "learning_rate": 9.462273839026624e-06,
+ "loss": 0.7242,
+ "step": 387
+ },
+ {
+ "epoch": 3.45,
+ "grad_norm": 0.36645383790153846,
+ "learning_rate": 9.159365321738655e-06,
+ "loss": 0.6773,
+ "step": 388
+ },
+ {
+ "epoch": 3.46,
+ "grad_norm": 0.29937594196593476,
+ "learning_rate": 8.861151583532656e-06,
+ "loss": 0.8557,
+ "step": 389
+ },
+ {
+ "epoch": 3.47,
+ "grad_norm": 0.382087527071458,
+ "learning_rate": 8.56764803653809e-06,
+ "loss": 0.8144,
+ "step": 390
+ },
+ {
+ "epoch": 3.48,
+ "grad_norm": 0.37165100019020914,
+ "learning_rate": 8.278869849454718e-06,
+ "loss": 0.8668,
+ "step": 391
+ },
+ {
+ "epoch": 3.49,
+ "grad_norm": 0.29949583615336506,
+ "learning_rate": 7.994831946768622e-06,
+ "loss": 0.6793,
+ "step": 392
+ },
+ {
+ "epoch": 3.49,
+ "eval_loss": 1.2786916494369507,
+ "eval_runtime": 13.2527,
+ "eval_samples_per_second": 22.637,
+ "eval_steps_per_second": 2.867,
+ "step": 392
+ },
+ {
+ "epoch": 3.5,
+ "grad_norm": 0.3128750696018338,
+ "learning_rate": 7.715549007981027e-06,
+ "loss": 0.7723,
+ "step": 393
+ },
+ {
+ "epoch": 3.51,
+ "grad_norm": 0.3036786519151417,
+ "learning_rate": 7.441035466849489e-06,
+ "loss": 0.7668,
+ "step": 394
+ },
+ {
+ "epoch": 3.52,
+ "grad_norm": 0.3723258995422837,
+ "learning_rate": 7.171305510642023e-06,
+ "loss": 0.7493,
+ "step": 395
+ },
+ {
+ "epoch": 3.53,
+ "grad_norm": 0.34640959843456987,
+ "learning_rate": 6.906373079403849e-06,
+ "loss": 0.7326,
+ "step": 396
+ },
+ {
+ "epoch": 3.53,
+ "grad_norm": 0.3708769315457807,
+ "learning_rate": 6.646251865236997e-06,
+ "loss": 0.7532,
+ "step": 397
+ },
+ {
+ "epoch": 3.54,
+ "grad_norm": 0.3204533685892251,
+ "learning_rate": 6.390955311592617e-06,
+ "loss": 0.7287,
+ "step": 398
+ },
+ {
+ "epoch": 3.55,
+ "grad_norm": 0.32625233919759344,
+ "learning_rate": 6.140496612576241e-06,
+ "loss": 0.7138,
+ "step": 399
+ },
+ {
+ "epoch": 3.56,
+ "grad_norm": 0.30531644707337335,
+ "learning_rate": 5.8948887122658335e-06,
+ "loss": 0.7123,
+ "step": 400
+ },
+ {
+ "epoch": 3.57,
+ "grad_norm": 0.33767502582039044,
+ "learning_rate": 5.65414430404293e-06,
+ "loss": 0.797,
+ "step": 401
+ },
+ {
+ "epoch": 3.58,
+ "grad_norm": 0.3282428030443849,
+ "learning_rate": 5.418275829936537e-06,
+ "loss": 0.8456,
+ "step": 402
+ },
+ {
+ "epoch": 3.59,
+ "grad_norm": 0.3532395206693705,
+ "learning_rate": 5.187295479980136e-06,
+ "loss": 0.7484,
+ "step": 403
+ },
+ {
+ "epoch": 3.6,
+ "grad_norm": 0.32556237019334716,
+ "learning_rate": 4.961215191581692e-06,
+ "loss": 0.7909,
+ "step": 404
+ },
+ {
+ "epoch": 3.61,
+ "grad_norm": 0.34045978047642994,
+ "learning_rate": 4.740046648906682e-06,
+ "loss": 0.7618,
+ "step": 405
+ },
+ {
+ "epoch": 3.61,
+ "grad_norm": 0.28937387449733615,
+ "learning_rate": 4.523801282274287e-06,
+ "loss": 0.8292,
+ "step": 406
+ },
+ {
+ "epoch": 3.62,
+ "grad_norm": 0.3160933371902765,
+ "learning_rate": 4.312490267566616e-06,
+ "loss": 0.7755,
+ "step": 407
+ },
+ {
+ "epoch": 3.63,
+ "grad_norm": 0.31618578023192107,
+ "learning_rate": 4.106124525651123e-06,
+ "loss": 0.8033,
+ "step": 408
+ },
+ {
+ "epoch": 3.64,
+ "grad_norm": 0.3105026069614601,
+ "learning_rate": 3.904714721816228e-06,
+ "loss": 0.7873,
+ "step": 409
+ },
+ {
+ "epoch": 3.65,
+ "grad_norm": 0.34709823487491576,
+ "learning_rate": 3.7082712652200867e-06,
+ "loss": 0.7882,
+ "step": 410
+ },
+ {
+ "epoch": 3.66,
+ "grad_norm": 0.3349684333518626,
+ "learning_rate": 3.516804308352628e-06,
+ "loss": 0.7841,
+ "step": 411
+ },
+ {
+ "epoch": 3.67,
+ "grad_norm": 0.3467993408892318,
+ "learning_rate": 3.330323746510955e-06,
+ "loss": 0.7605,
+ "step": 412
+ },
+ {
+ "epoch": 3.68,
+ "grad_norm": 0.37371307324588393,
+ "learning_rate": 3.148839217287758e-06,
+ "loss": 0.7229,
+ "step": 413
+ },
+ {
+ "epoch": 3.69,
+ "grad_norm": 0.32271590708905057,
+ "learning_rate": 2.9723601000734016e-06,
+ "loss": 0.7227,
+ "step": 414
+ },
+ {
+ "epoch": 3.69,
+ "grad_norm": 0.3080800887837192,
+ "learning_rate": 2.800895515571078e-06,
+ "loss": 0.7322,
+ "step": 415
+ },
+ {
+ "epoch": 3.7,
+ "grad_norm": 0.3563938469507623,
+ "learning_rate": 2.634454325325497e-06,
+ "loss": 0.7951,
+ "step": 416
+ },
+ {
+ "epoch": 3.71,
+ "grad_norm": 0.3544288347449977,
+ "learning_rate": 2.473045131264862e-06,
+ "loss": 0.7204,
+ "step": 417
+ },
+ {
+ "epoch": 3.72,
+ "grad_norm": 0.37683581963069984,
+ "learning_rate": 2.3166762752563463e-06,
+ "loss": 0.7817,
+ "step": 418
+ },
+ {
+ "epoch": 3.73,
+ "grad_norm": 0.33996733249074595,
+ "learning_rate": 2.1653558386749427e-06,
+ "loss": 0.68,
+ "step": 419
+ },
+ {
+ "epoch": 3.74,
+ "grad_norm": 0.3069725548353936,
+ "learning_rate": 2.0190916419858484e-06,
+ "loss": 0.798,
+ "step": 420
+ },
+ {
+ "epoch": 3.74,
+ "eval_loss": 1.280185580253601,
+ "eval_runtime": 13.1908,
+ "eval_samples_per_second": 22.743,
+ "eval_steps_per_second": 2.881,
+ "step": 420
+ },
+ {
+ "epoch": 3.75,
+ "grad_norm": 0.3417039799474373,
+ "learning_rate": 1.8778912443402242e-06,
+ "loss": 0.7972,
+ "step": 421
+ },
+ {
+ "epoch": 3.76,
+ "grad_norm": 0.36875255309045446,
+ "learning_rate": 1.7417619431845944e-06,
+ "loss": 0.6653,
+ "step": 422
+ },
+ {
+ "epoch": 3.77,
+ "grad_norm": 0.37727682021595976,
+ "learning_rate": 1.6107107738836835e-06,
+ "loss": 0.7994,
+ "step": 423
+ },
+ {
+ "epoch": 3.77,
+ "grad_norm": 0.3680295622641893,
+ "learning_rate": 1.4847445093567836e-06,
+ "loss": 0.7551,
+ "step": 424
+ },
+ {
+ "epoch": 3.78,
+ "grad_norm": 0.34533281482301054,
+ "learning_rate": 1.3638696597277679e-06,
+ "loss": 0.7416,
+ "step": 425
+ },
+ {
+ "epoch": 3.79,
+ "grad_norm": 0.3908807108284709,
+ "learning_rate": 1.2480924719885934e-06,
+ "loss": 0.7188,
+ "step": 426
+ },
+ {
+ "epoch": 3.8,
+ "grad_norm": 0.31916637760320243,
+ "learning_rate": 1.1374189296765037e-06,
+ "loss": 0.7594,
+ "step": 427
+ },
+ {
+ "epoch": 3.81,
+ "grad_norm": 0.33316597860051206,
+ "learning_rate": 1.0318547525647315e-06,
+ "loss": 0.7249,
+ "step": 428
+ },
+ {
+ "epoch": 3.82,
+ "grad_norm": 0.3416859037907314,
+ "learning_rate": 9.314053963669245e-07,
+ "loss": 0.8137,
+ "step": 429
+ },
+ {
+ "epoch": 3.83,
+ "grad_norm": 0.3166447199987983,
+ "learning_rate": 8.360760524551814e-07,
+ "loss": 0.7548,
+ "step": 430
+ },
+ {
+ "epoch": 3.84,
+ "grad_norm": 0.3678155762471668,
+ "learning_rate": 7.458716475917559e-07,
+ "loss": 0.6848,
+ "step": 431
+ },
+ {
+ "epoch": 3.85,
+ "grad_norm": 0.3564576656343474,
+ "learning_rate": 6.607968436744272e-07,
+ "loss": 0.7188,
+ "step": 432
+ },
+ {
+ "epoch": 3.85,
+ "grad_norm": 0.31739334088982646,
+ "learning_rate": 5.808560374955585e-07,
+ "loss": 0.6899,
+ "step": 433
+ },
+ {
+ "epoch": 3.86,
+ "grad_norm": 0.3517404847481434,
+ "learning_rate": 5.06053360514902e-07,
+ "loss": 0.7085,
+ "step": 434
+ },
+ {
+ "epoch": 3.87,
+ "grad_norm": 0.32714795923909773,
+ "learning_rate": 4.363926786460359e-07,
+ "loss": 0.8216,
+ "step": 435
+ },
+ {
+ "epoch": 3.88,
+ "grad_norm": 0.3635146713982181,
+ "learning_rate": 3.718775920565687e-07,
+ "loss": 0.7684,
+ "step": 436
+ },
+ {
+ "epoch": 3.89,
+ "grad_norm": 0.3419976870684463,
+ "learning_rate": 3.125114349821212e-07,
+ "loss": 0.7825,
+ "step": 437
+ },
+ {
+ "epoch": 3.9,
+ "grad_norm": 0.3872335456149991,
+ "learning_rate": 2.5829727555397584e-07,
+ "loss": 0.8252,
+ "step": 438
+ },
+ {
+ "epoch": 3.91,
+ "grad_norm": 0.3918700023496713,
+ "learning_rate": 2.0923791564050333e-07,
+ "loss": 0.6846,
+ "step": 439
+ },
+ {
+ "epoch": 3.92,
+ "grad_norm": 0.3039893808886515,
+ "learning_rate": 1.6533589070237832e-07,
+ "loss": 0.6893,
+ "step": 440
+ },
+ {
+ "epoch": 3.93,
+ "grad_norm": 0.2983095993470017,
+ "learning_rate": 1.2659346966152895e-07,
+ "loss": 0.7482,
+ "step": 441
+ },
+ {
+ "epoch": 3.93,
+ "grad_norm": 0.29173707821708994,
+ "learning_rate": 9.30126547838861e-08,
+ "loss": 0.8286,
+ "step": 442
+ },
+ {
+ "epoch": 3.94,
+ "grad_norm": 0.3410941283765189,
+ "learning_rate": 6.459518157591048e-08,
+ "loss": 0.7608,
+ "step": 443
+ },
+ {
+ "epoch": 3.95,
+ "grad_norm": 0.3551291216161622,
+ "learning_rate": 4.1342518694853506e-08,
+ "loss": 0.8346,
+ "step": 444
+ },
+ {
+ "epoch": 3.96,
+ "grad_norm": 0.340830105966026,
+ "learning_rate": 2.3255867872928882e-08,
+ "loss": 0.7388,
+ "step": 445
+ },
+ {
+ "epoch": 3.97,
+ "grad_norm": 0.3072115448848053,
+ "learning_rate": 1.0336163855129144e-08,
+ "loss": 0.842,
+ "step": 446
+ },
+ {
+ "epoch": 3.98,
+ "grad_norm": 0.3339675653700237,
+ "learning_rate": 2.5840743509863983e-09,
+ "loss": 0.7289,
+ "step": 447
+ },
+ {
+ "epoch": 3.99,
+ "grad_norm": 0.29581508154823644,
+ "learning_rate": 0.0,
+ "loss": 0.7461,
+ "step": 448
+ },
+ {
+ "epoch": 3.99,
+ "eval_loss": 1.2807621955871582,
+ "eval_runtime": 13.2308,
+ "eval_samples_per_second": 22.674,
+ "eval_steps_per_second": 2.872,
+ "step": 448
+ }
+ ],
+ "logging_steps": 1,
+ "max_steps": 448,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 4,
+ "save_steps": 112,
+ "total_flos": 5.340598301349642e+17,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/checkpoint-448/training_args.bin b/checkpoint-448/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4992a3b445f8065303289f8a30d4fedc43285059
--- /dev/null
+++ b/checkpoint-448/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d400c16f982c36b10268ff7e69e878c44d11f5fb692a61770a8e1efb50d4491c
+size 6776
diff --git a/checkpoint-448/zero_to_fp32.py b/checkpoint-448/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..49b846633d6eb1e836e34681e44033581f4edb7b
--- /dev/null
+++ b/checkpoint-448/zero_to_fp32.py
@@ -0,0 +1,592 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+ total_files = len(files)
+ state_dicts = []
+ for f in files:
+ state_dict = torch.load(f, map_location=device)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ if zero_stage <= 2:
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ elif zero_stage == 3:
+ # if there is more than one param group, there will be multiple flattened tensors - one
+ # flattened tensor per group - for simplicity merge them into a single tensor
+ #
+ # XXX: could make the script more memory efficient for when there are multiple groups - it
+ # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+ fp32_flat_groups = [
+ torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+ ]
+
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ model_files = get_model_state_files(ds_checkpoint_dir)
+
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ for name, shape in param_shapes.items():
+
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # XXX: memory usage doubles here
+ state_dict[name] = torch.cat(
+ tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+ 0).narrow(0, 0, unpartitioned_numel).view(shape)
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+ you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ """
+
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+ print(f"Saving fp32 state dict to {output_file}")
+ torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info(f"Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info(f"Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument(
+ "output_file",
+ type=str,
+ help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..8259cac845aa2eaa55cc7cc08d2d4de1aae197ec
--- /dev/null
+++ b/config.json
@@ -0,0 +1,43 @@
+{
+ "_name_or_path": "google/gemma-7b-it",
+ "architectures": [
+ "GemmaForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 2,
+ "eos_token_id": 1,
+ "head_dim": 256,
+ "hidden_act": "gelu",
+ "hidden_size": 3072,
+ "initializer_range": 0.02,
+ "intermediate_size": 24576,
+ "max_position_embeddings": 8192,
+ "model_type": "gemma",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 16,
+ "pad_token_id": 0,
+ "quantization_config": {
+ "_load_in_4bit": true,
+ "_load_in_8bit": false,
+ "bnb_4bit_compute_dtype": "bfloat16",
+ "bnb_4bit_quant_storage": "uint8",
+ "bnb_4bit_quant_type": "nf4",
+ "bnb_4bit_use_double_quant": true,
+ "llm_int8_enable_fp32_cpu_offload": false,
+ "llm_int8_has_fp16_weight": false,
+ "llm_int8_skip_modules": null,
+ "llm_int8_threshold": 6.0,
+ "load_in_4bit": true,
+ "load_in_8bit": false,
+ "quant_method": "bitsandbytes"
+ },
+ "rms_norm_eps": 1e-06,
+ "rope_scaling": null,
+ "rope_theta": 10000.0,
+ "torch_dtype": "bfloat16",
+ "transformers_version": "4.39.0.dev0",
+ "use_cache": false,
+ "vocab_size": 256000
+}
diff --git a/merged/config.json b/merged/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5cf635c5516ca060a95e347a8fd1d8b0e1719cbf
--- /dev/null
+++ b/merged/config.json
@@ -0,0 +1,28 @@
+{
+ "_name_or_path": "google/gemma-2b",
+ "architectures": [
+ "GemmaForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 2,
+ "eos_token_id": 1,
+ "head_dim": 256,
+ "hidden_act": "gelu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 16384,
+ "max_position_embeddings": 8192,
+ "model_type": "gemma",
+ "num_attention_heads": 8,
+ "num_hidden_layers": 18,
+ "num_key_value_heads": 1,
+ "pad_token_id": 0,
+ "rms_norm_eps": 1e-06,
+ "rope_scaling": null,
+ "rope_theta": 10000.0,
+ "torch_dtype": "bfloat16",
+ "transformers_version": "4.39.0.dev0",
+ "use_cache": false,
+ "vocab_size": 256000
+}
diff --git a/merged/generation_config.json b/merged/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6315b520e1541d5ac8d7c9fa201d06bdc5cc22ce
--- /dev/null
+++ b/merged/generation_config.json
@@ -0,0 +1,8 @@
+{
+ "_from_model_config": true,
+ "bos_token_id": 2,
+ "do_sample": true,
+ "eos_token_id": 1,
+ "pad_token_id": 0,
+ "transformers_version": "4.39.0.dev0"
+}
diff --git a/merged/pytorch_model-00001-of-00002.bin b/merged/pytorch_model-00001-of-00002.bin
new file mode 100644
index 0000000000000000000000000000000000000000..181a1825d468da8a4d705495b4c3fc068731c5f0
--- /dev/null
+++ b/merged/pytorch_model-00001-of-00002.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd397dcd6cec61c08e4380b576faa8636353e8bb163f5fee34d9a4daf0ad10c3
+size 4945278526
diff --git a/merged/pytorch_model-00002-of-00002.bin b/merged/pytorch_model-00002-of-00002.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1ee0c70a98a1a349b85746d380c598818722b2c3
--- /dev/null
+++ b/merged/pytorch_model-00002-of-00002.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:142db532ccdbd1654a08d9f48be8e8790be8660dd96a131697cadbd56691733c
+size 67123448
diff --git a/merged/pytorch_model.bin.index.json b/merged/pytorch_model.bin.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..e4d9d71c0ce85807156139e7d4e24f1274f10700
--- /dev/null
+++ b/merged/pytorch_model.bin.index.json
@@ -0,0 +1,172 @@
+{
+ "metadata": {
+ "total_size": 5012344832
+ },
+ "weight_map": {
+ "lm_head.weight": "pytorch_model-00001-of-00002.bin",
+ "model.embed_tokens.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+ "model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+ "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.17.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+ "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+ "model.norm.weight": "pytorch_model-00002-of-00002.bin"
+ }
+}
diff --git a/merged/special_tokens_map.json b/merged/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6119589e367b2de0fc8cbd2f1217667532e3174
--- /dev/null
+++ b/merged/special_tokens_map.json
@@ -0,0 +1,30 @@
+{
+ "bos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "eos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "pad_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "unk_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ }
+}
diff --git a/merged/tokenizer.json b/merged/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..7d526fa4a2fc483e768773df90a944b54094e367
--- /dev/null
+++ b/merged/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0d908b4f9326e0998815690e325b6abbd378978553e10627924dd825db7e243
+size 17477553
diff --git a/merged/tokenizer.model b/merged/tokenizer.model
new file mode 100644
index 0000000000000000000000000000000000000000..796efe9ab515c15e146ce7588e6d7b9b8134dbf8
--- /dev/null
+++ b/merged/tokenizer.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003
diff --git a/merged/tokenizer_config.json b/merged/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..c22f33ac79a1ea3d99aa4e2c78d2379d98747f72
--- /dev/null
+++ b/merged/tokenizer_config.json
@@ -0,0 +1,49 @@
+{
+ "add_bos_token": true,
+ "add_eos_token": false,
+ "added_tokens_decoder": {
+ "0": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "1": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "2": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "3": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "bos_token": "",
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "",
+ "legacy": null,
+ "model_max_length": 1000000000000000019884624838656,
+ "pad_token": "",
+ "sp_model_kwargs": {},
+ "spaces_between_special_tokens": false,
+ "tokenizer_class": "GemmaTokenizer",
+ "unk_token": "",
+ "use_default_system_prompt": false
+}
diff --git a/runs/Mar21_07-07-55_8205afe3ecd2/events.out.tfevents.1711004877.8205afe3ecd2.2618.0 b/runs/Mar21_07-07-55_8205afe3ecd2/events.out.tfevents.1711004877.8205afe3ecd2.2618.0
new file mode 100644
index 0000000000000000000000000000000000000000..30668ca1217577b22669b6e1fdd436ce13067833
--- /dev/null
+++ b/runs/Mar21_07-07-55_8205afe3ecd2/events.out.tfevents.1711004877.8205afe3ecd2.2618.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d38690a2d000335f0399b91add00dcb371cb0a32b1af3354dc3bb1227c471f2
+size 104837
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..8d6368f7e735fbe4781bf6e956b7c6ad0586df80
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,34 @@
+{
+ "additional_special_tokens": [
+ "",
+ ""
+ ],
+ "bos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "eos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "pad_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "unk_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ }
+}
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..10d1ca087d5e78cbfc41ae083201df86aa031044
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05e97791a5e007260de1db7e1692e53150e08cea481e2bf25435553380c147ee
+size 17477929
diff --git a/tokenizer.model b/tokenizer.model
new file mode 100644
index 0000000000000000000000000000000000000000..796efe9ab515c15e146ce7588e6d7b9b8134dbf8
--- /dev/null
+++ b/tokenizer.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..1451ca400a880c5056ab40cb960930cf827de2de
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,70 @@
+{
+ "add_bos_token": true,
+ "add_eos_token": false,
+ "added_tokens_decoder": {
+ "0": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "1": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "2": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "3": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "106": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "107": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [
+ "",
+ ""
+ ],
+ "bos_token": "",
+ "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\n' + message['content'] | trim + '\n' }}{% endfor %}{% if add_generation_prompt %}{{'model\n'}}{% endif %}",
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "",
+ "legacy": null,
+ "model_max_length": 1000000000000000019884624838656,
+ "pad_token": "",
+ "sp_model_kwargs": {},
+ "spaces_between_special_tokens": false,
+ "tokenizer_class": "GemmaTokenizer",
+ "unk_token": "",
+ "use_default_system_prompt": false
+}