Add files using upload-large-folder tool

README.md

@@ -0,0 +1,164 @@
+---
+library_name: peft
+license: llama2
+base_model: codellama/CodeLlama-7b-hf
+tags:
+- axolotl
+- base_model:adapter:codellama/CodeLlama-7b-hf
+- lora
+- transformers
+datasets:
+- darwinkernelpanic/luau-reasoning-normalized
+pipeline_tag: text-generation
+model-index:
+- name: outputs/luau-codellama-h200-fast
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.13.0.dev0`
+```yaml
+base_model: codellama/CodeLlama-7b-hf
+model_type: LlamaForCausalLM
+tokenizer_type: LlamaTokenizer
+
+# Keep full precision weights (fast on Hopper)
+load_in_8bit: false
+load_in_4bit: false
+strict: false
+
+chat_template: llama3
+
+datasets:
+  - path: darwinkernelpanic/luau-reasoning-normalized
+    type: chat_template
+    conversation: llama3
+    field_messages: messages
+    add_generation_prompt: true
+
+# Preprocessing workers (CPU). Fine as-is.
+num_proc: 16
+
+output_dir: ./outputs/luau-codellama-h200-fast
+
+# ===== LoRA =====
+adapter: lora
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.05
+lora_target_modules:
+  - q_proj
+  - k_proj
+  - v_proj
+  - o_proj
+
+# ===== Precision =====
+bf16: true
+fp16: false
+tf32: true
+
+# ===== Sequence / batching =====
+sequence_len: 4096
+# Keep packing for throughput, but enable length grouping to cut padding
+sample_packing: true
+group_by_length: true
+
+# Lower micro-batch a bit to kill peak VRAM while staying fast
+micro_batch_size: 5
+gradient_accumulation_steps: 1
+
+# ===== Training =====
+num_epochs: 3
+optimizer: adamw_torch
+learning_rate: 2e-4
+lr_scheduler_type: cosine
+warmup_steps: 100
+
+train_on_inputs: false
+
+# Turn on checkpointing — tiny speed hit, big memory win
+gradient_checkpointing: true
+gradient_clipping: 1.0
+
+# ===== Dataloader =====
+# Keep pin_memory, but avoid too many loader workers in Accelerate
+dataloader_num_workers: 2
+dataloader_pin_memory: true
+# Optional: avoid insanely large host->device prefetch
+# dataloader_prefetch_factor: 2
+
+# ===== Logging / eval =====
+logging_steps: 25
+val_set_size: 0.05
+# Reduce eval/save frequency to avoid spikes
+eval_steps: 1000
+save_strategy: steps
+save_steps: 1000
+save_total_limit: 3
+
+seed: 42
+
+# ===== DeepSpeed =====
+# Off for single H200 — overhead not worth it for 7B
+```
+
+</details><br>
+
+# outputs/luau-codellama-h200-fast
+
+This model is a fine-tuned version of [codellama/CodeLlama-7b-hf](https://huggingface.co/codellama/CodeLlama-7b-hf) on the darwinkernelpanic/luau-reasoning-normalized dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4927
+- Ppl: 1.6368
+- Memory/max Active (gib): 19.1
+- Memory/max Allocated (gib): 19.1
+- Memory/device Reserved (gib): 139.06
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 5
+- eval_batch_size: 5
+- seed: 42
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 100
+- training_steps: 3996
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Ppl    | Active (gib) | Allocated (gib) | Reserved (gib) |
+|:-------------:|:------:|:----:|:---------------:|:------:|:------------:|:---------------:|:--------------:|
+| No log        | 0      | 0    | 1.6888          | 5.4129 | 18.94        | 18.94           | 139.12         |
+| 0.5511        | 0.7502 | 1000 | 0.5410          | 1.7177 | 19.1         | 19.1            | 139.02         |
+| 0.5052        | 1.5004 | 2000 | 0.5064          | 1.6593 | 19.1         | 19.1            | 139.06         |
+| 0.4733        | 2.2506 | 3000 | 0.4927          | 1.6368 | 19.1         | 19.1            | 139.06         |
+
+
+### Framework versions
+
+- PEFT 0.18.0
+- Transformers 4.57.1
+- Pytorch 2.8.0+cu128
+- Datasets 4.4.1
+- Tokenizers 0.22.1

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5435b9255c61a5d527452354bcf1f8d19766555d80a880c8058888a3d14d32e4
-size 33589040
+oid sha256:117f1345e5c50f25128a7a46faf63217f9c93561ea40a0465fc49fe91eab762a
+size 67143296

checkpoint-2000/README.md

@@ -0,0 +1,208 @@
+---
+base_model: codellama/CodeLlama-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:codellama/CodeLlama-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.0

checkpoint-2000/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "codellama/CodeLlama-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-2000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66a52b1a3b7e275e6ca6a1b4e9c618cadb590db898823dfe8668c304684e5b78
+size 67143296

checkpoint-2000/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42575df9ba3d20ffae7081ec06b874a4a485aef3188b2c57e317c3014149df80
+size 134433995

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dd1989f8fb25657391bdb38e58849515e95e29f553d1fb6f5355a4baca32f48
+size 14645

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc5fa3ab2e76e63a76d96d1e2420300bea66d7082b0e604c6c4f6b71e904105a
+size 1465

checkpoint-2000/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ccb9c8b6b561889acea59191d66986d314e7cbd6a78abc6e49b139ca91c1e6
+size 500058

checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

checkpoint-2000/trainer_state.json

@@ -0,0 +1,1110 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.5003750937734432,
+  "eval_steps": 1000,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0,
+      "eval_loss": 1.6887853145599365,
+      "eval_ppl": 5.4129,
+      "eval_runtime": 167.3526,
+      "eval_samples_per_second": 4.362,
+      "eval_steps_per_second": 0.872,
+      "memory/device_reserved (GiB)": 139.12,
+      "memory/max_active (GiB)": 18.94,
+      "memory/max_allocated (GiB)": 18.94,
+      "step": 0
+    },
+    {
+      "epoch": 0.018754688672168042,
+      "grad_norm": 1.415561556816101,
+      "learning_rate": 4.8e-05,
+      "loss": 1.6848,
+      "memory/device_reserved (GiB)": 139.11,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 5.3914,
+      "step": 25,
+      "tokens_per_second_per_gpu": 16277.76,
+      "total_tokens": 1723633
+    },
+    {
+      "epoch": 0.037509377344336084,
+      "grad_norm": 0.33179354667663574,
+      "learning_rate": 9.8e-05,
+      "loss": 0.9839,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.6749,
+      "step": 50,
+      "tokens_per_second_per_gpu": 4303.21,
+      "total_tokens": 2175386
+    },
+    {
+      "epoch": 0.056264066016504126,
+      "grad_norm": 0.17453454434871674,
+      "learning_rate": 0.000148,
+      "loss": 0.8002,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.226,
+      "step": 75,
+      "tokens_per_second_per_gpu": 3776.03,
+      "total_tokens": 2623712
+    },
+    {
+      "epoch": 0.07501875468867217,
+      "grad_norm": 0.19318008422851562,
+      "learning_rate": 0.00019800000000000002,
+      "loss": 0.7218,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.0581,
+      "step": 100,
+      "tokens_per_second_per_gpu": 4252.49,
+      "total_tokens": 3072519
+    },
+    {
+      "epoch": 0.09377344336084022,
+      "grad_norm": 0.18435686826705933,
+      "learning_rate": 0.00019998127418269004,
+      "loss": 0.6759,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.9658,
+      "step": 125,
+      "tokens_per_second_per_gpu": 4303.31,
+      "total_tokens": 3523983
+    },
+    {
+      "epoch": 0.11252813203300825,
+      "grad_norm": 0.19870473444461823,
+      "learning_rate": 0.00019992195096972548,
+      "loss": 0.6703,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.9548,
+      "step": 150,
+      "tokens_per_second_per_gpu": 4260.86,
+      "total_tokens": 3973452
+    },
+    {
+      "epoch": 0.1312828207051763,
+      "grad_norm": 0.20499658584594727,
+      "learning_rate": 0.0001998220219574743,
+      "loss": 0.6381,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8929,
+      "step": 175,
+      "tokens_per_second_per_gpu": 4288.64,
+      "total_tokens": 4423763
+    },
+    {
+      "epoch": 0.15003750937734434,
+      "grad_norm": 0.18934418261051178,
+      "learning_rate": 0.00019968152775460537,
+      "loss": 0.6383,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8933,
+      "step": 200,
+      "tokens_per_second_per_gpu": 4244.79,
+      "total_tokens": 4872365
+    },
+    {
+      "epoch": 0.16879219804951237,
+      "grad_norm": 0.1827855408191681,
+      "learning_rate": 0.00019950052545447352,
+      "loss": 0.6347,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8865,
+      "step": 225,
+      "tokens_per_second_per_gpu": 4252.71,
+      "total_tokens": 5319322
+    },
+    {
+      "epoch": 0.18754688672168043,
+      "grad_norm": 0.16483066976070404,
+      "learning_rate": 0.00019927908861191827,
+      "loss": 0.6392,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.895,
+      "step": 250,
+      "tokens_per_second_per_gpu": 3772.0,
+      "total_tokens": 5768644
+    },
+    {
+      "epoch": 0.20630157539384847,
+      "grad_norm": 0.17186357080936432,
+      "learning_rate": 0.00019901730721337302,
+      "loss": 0.614,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8478,
+      "step": 275,
+      "tokens_per_second_per_gpu": 4281.82,
+      "total_tokens": 6220751
+    },
+    {
+      "epoch": 0.2250562640660165,
+      "grad_norm": 0.18073013424873352,
+      "learning_rate": 0.00019871528764029667,
+      "loss": 0.6196,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8582,
+      "step": 300,
+      "tokens_per_second_per_gpu": 4234.51,
+      "total_tokens": 6668111
+    },
+    {
+      "epoch": 0.24381095273818454,
+      "grad_norm": 0.19639697670936584,
+      "learning_rate": 0.00019837315262594306,
+      "loss": 0.6181,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8554,
+      "step": 325,
+      "tokens_per_second_per_gpu": 4261.44,
+      "total_tokens": 7117439
+    },
+    {
+      "epoch": 0.2625656414103526,
+      "grad_norm": 0.1670486479997635,
+      "learning_rate": 0.00019799104120548492,
+      "loss": 0.6141,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.848,
+      "step": 350,
+      "tokens_per_second_per_gpu": 4298.97,
+      "total_tokens": 7569060
+    },
+    {
+      "epoch": 0.2813203300825206,
+      "grad_norm": 0.17752495408058167,
+      "learning_rate": 0.00019756910865951377,
+      "loss": 0.6075,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8358,
+      "step": 375,
+      "tokens_per_second_per_gpu": 4256.6,
+      "total_tokens": 8017630
+    },
+    {
+      "epoch": 0.30007501875468867,
+      "grad_norm": 0.2000180035829544,
+      "learning_rate": 0.00019710752645093747,
+      "loss": 0.6108,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8419,
+      "step": 400,
+      "tokens_per_second_per_gpu": 4245.46,
+      "total_tokens": 8464998
+    },
+    {
+      "epoch": 0.31882970742685673,
+      "grad_norm": 0.17395919561386108,
+      "learning_rate": 0.00019660648215530206,
+      "loss": 0.5966,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8159,
+      "step": 425,
+      "tokens_per_second_per_gpu": 3758.92,
+      "total_tokens": 8914723
+    },
+    {
+      "epoch": 0.33758439609902474,
+      "grad_norm": 0.18785236775875092,
+      "learning_rate": 0.00019606617938456572,
+      "loss": 0.6099,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8402,
+      "step": 450,
+      "tokens_per_second_per_gpu": 4200.53,
+      "total_tokens": 9359638
+    },
+    {
+      "epoch": 0.3563390847711928,
+      "grad_norm": 0.17702797055244446,
+      "learning_rate": 0.0001954868377043559,
+      "loss": 0.5922,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.808,
+      "step": 475,
+      "tokens_per_second_per_gpu": 4265.36,
+      "total_tokens": 9810837
+    },
+    {
+      "epoch": 0.37509377344336087,
+      "grad_norm": 0.19927558302879333,
+      "learning_rate": 0.00019486869254474337,
+      "loss": 0.5759,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7787,
+      "step": 500,
+      "tokens_per_second_per_gpu": 4276.25,
+      "total_tokens": 10261446
+    },
+    {
+      "epoch": 0.3938484621155289,
+      "grad_norm": 0.1908370852470398,
+      "learning_rate": 0.0001942119951045692,
+      "loss": 0.584,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7932,
+      "step": 525,
+      "tokens_per_second_per_gpu": 4272.28,
+      "total_tokens": 10707841
+    },
+    {
+      "epoch": 0.41260315078769694,
+      "grad_norm": 0.2064146101474762,
+      "learning_rate": 0.00019351701224936383,
+      "loss": 0.5791,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7844,
+      "step": 550,
+      "tokens_per_second_per_gpu": 4250.37,
+      "total_tokens": 11155384
+    },
+    {
+      "epoch": 0.43135783945986494,
+      "grad_norm": 0.26748332381248474,
+      "learning_rate": 0.0001927840264028995,
+      "loss": 0.5758,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7786,
+      "step": 575,
+      "tokens_per_second_per_gpu": 4256.55,
+      "total_tokens": 11601192
+    },
+    {
+      "epoch": 0.450112528132033,
+      "grad_norm": 0.17514832317829132,
+      "learning_rate": 0.00019201333543242036,
+      "loss": 0.5791,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7844,
+      "step": 600,
+      "tokens_per_second_per_gpu": 3770.83,
+      "total_tokens": 12048477
+    },
+    {
+      "epoch": 0.46886721680420107,
+      "grad_norm": 0.22069169580936432,
+      "learning_rate": 0.00019120525252759647,
+      "loss": 0.5803,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7866,
+      "step": 625,
+      "tokens_per_second_per_gpu": 4179.31,
+      "total_tokens": 12488141
+    },
+    {
+      "epoch": 0.4876219054763691,
+      "grad_norm": 0.20555566251277924,
+      "learning_rate": 0.00019036010607325138,
+      "loss": 0.5716,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7711,
+      "step": 650,
+      "tokens_per_second_per_gpu": 4209.96,
+      "total_tokens": 12934358
+    },
+    {
+      "epoch": 0.5063765941485371,
+      "grad_norm": 0.19018156826496124,
+      "learning_rate": 0.00018947823951591478,
+      "loss": 0.5608,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7521,
+      "step": 675,
+      "tokens_per_second_per_gpu": 4226.4,
+      "total_tokens": 13378983
+    },
+    {
+      "epoch": 0.5251312828207052,
+      "grad_norm": 0.17173859477043152,
+      "learning_rate": 0.00018856001122425416,
+      "loss": 0.5667,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7624,
+      "step": 700,
+      "tokens_per_second_per_gpu": 4265.57,
+      "total_tokens": 13829519
+    },
+    {
+      "epoch": 0.5438859714928732,
+      "grad_norm": 0.17706550657749176,
+      "learning_rate": 0.0001876057943434428,
+      "loss": 0.565,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7594,
+      "step": 725,
+      "tokens_per_second_per_gpu": 4281.61,
+      "total_tokens": 14281879
+    },
+    {
+      "epoch": 0.5626406601650412,
+      "grad_norm": 0.18528586626052856,
+      "learning_rate": 0.00018661597664352284,
+      "loss": 0.5666,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7623,
+      "step": 750,
+      "tokens_per_second_per_gpu": 4229.32,
+      "total_tokens": 14725919
+    },
+    {
+      "epoch": 0.5813953488372093,
+      "grad_norm": 0.16790929436683655,
+      "learning_rate": 0.00018559096036182516,
+      "loss": 0.5633,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7565,
+      "step": 775,
+      "tokens_per_second_per_gpu": 3775.0,
+      "total_tokens": 15175146
+    },
+    {
+      "epoch": 0.6001500375093773,
+      "grad_norm": 0.17511805891990662,
+      "learning_rate": 0.00018453116203951005,
+      "loss": 0.5664,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7619,
+      "step": 800,
+      "tokens_per_second_per_gpu": 4218.07,
+      "total_tokens": 15619901
+    },
+    {
+      "epoch": 0.6189047261815454,
+      "grad_norm": 0.19853387773036957,
+      "learning_rate": 0.0001834370123522954,
+      "loss": 0.5646,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7587,
+      "step": 825,
+      "tokens_per_second_per_gpu": 4230.84,
+      "total_tokens": 16066102
+    },
+    {
+      "epoch": 0.6376594148537135,
+      "grad_norm": 0.18872258067131042,
+      "learning_rate": 0.00018230895593544056,
+      "loss": 0.552,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7367,
+      "step": 850,
+      "tokens_per_second_per_gpu": 4222.33,
+      "total_tokens": 16510696
+    },
+    {
+      "epoch": 0.6564141035258815,
+      "grad_norm": 0.9702818989753723,
+      "learning_rate": 0.0001811474512030578,
+      "loss": 0.5607,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7519,
+      "step": 875,
+      "tokens_per_second_per_gpu": 4200.39,
+      "total_tokens": 16953918
+    },
+    {
+      "epoch": 0.6751687921980495,
+      "grad_norm": 0.17479568719863892,
+      "learning_rate": 0.00017995297016182405,
+      "loss": 0.564,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7577,
+      "step": 900,
+      "tokens_per_second_per_gpu": 4210.15,
+      "total_tokens": 17396453
+    },
+    {
+      "epoch": 0.6939234808702176,
+      "grad_norm": 0.1948954463005066,
+      "learning_rate": 0.0001787259982191692,
+      "loss": 0.5511,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7352,
+      "step": 925,
+      "tokens_per_second_per_gpu": 4237.98,
+      "total_tokens": 17841287
+    },
+    {
+      "epoch": 0.7126781695423856,
+      "grad_norm": 0.19541053473949432,
+      "learning_rate": 0.00017746703398601872,
+      "loss": 0.5532,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7388,
+      "step": 950,
+      "tokens_per_second_per_gpu": 3725.33,
+      "total_tokens": 18283596
+    },
+    {
+      "epoch": 0.7314328582145536,
+      "grad_norm": 0.1818365603685379,
+      "learning_rate": 0.0001761765890741701,
+      "loss": 0.5521,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7369,
+      "step": 975,
+      "tokens_per_second_per_gpu": 4211.63,
+      "total_tokens": 18726722
+    },
+    {
+      "epoch": 0.7501875468867217,
+      "grad_norm": 0.1838025599718094,
+      "learning_rate": 0.00017485518788838705,
+      "loss": 0.5511,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7352,
+      "step": 1000,
+      "tokens_per_second_per_gpu": 3962.4,
+      "total_tokens": 19167258
+    },
+    {
+      "epoch": 0.7501875468867217,
+      "eval_loss": 0.540988564491272,
+      "eval_ppl": 1.7177,
+      "eval_runtime": 138.0264,
+      "eval_samples_per_second": 5.289,
+      "eval_steps_per_second": 1.058,
+      "memory/device_reserved (GiB)": 139.02,
+      "memory/max_active (GiB)": 19.1,
+      "memory/max_allocated (GiB)": 19.1,
+      "step": 1000
+    },
+    {
+      "epoch": 0.7689422355588897,
+      "grad_norm": 0.2199818342924118,
+      "learning_rate": 0.00017350336741329413,
+      "loss": 0.549,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7315,
+      "step": 1025,
+      "tokens_per_second_per_gpu": 4129.73,
+      "total_tokens": 20870820
+    },
+    {
+      "epoch": 0.7876969242310577,
+      "grad_norm": 0.19783177971839905,
+      "learning_rate": 0.0001721216769951596,
+      "loss": 0.5615,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7533,
+      "step": 1050,
+      "tokens_per_second_per_gpu": 4243.63,
+      "total_tokens": 21317982
+    },
+    {
+      "epoch": 0.8064516129032258,
+      "grad_norm": 0.1678430140018463,
+      "learning_rate": 0.00017071067811865476,
+      "loss": 0.5557,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7432,
+      "step": 1075,
+      "tokens_per_second_per_gpu": 4092.04,
+      "total_tokens": 21754087
+    },
+    {
+      "epoch": 0.8252063015753939,
+      "grad_norm": 0.16523879766464233,
+      "learning_rate": 0.00016927094417868048,
+      "loss": 0.556,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7437,
+      "step": 1100,
+      "tokens_per_second_per_gpu": 4187.02,
+      "total_tokens": 22198779
+    },
+    {
+      "epoch": 0.8439609902475619,
+      "grad_norm": 0.18177717924118042,
+      "learning_rate": 0.00016780306024735382,
+      "loss": 0.5468,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7277,
+      "step": 1125,
+      "tokens_per_second_per_gpu": 4198.97,
+      "total_tokens": 22639769
+    },
+    {
+      "epoch": 0.8627156789197299,
+      "grad_norm": 0.17299720644950867,
+      "learning_rate": 0.0001663076228362492,
+      "loss": 0.554,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7402,
+      "step": 1150,
+      "tokens_per_second_per_gpu": 3762.13,
+      "total_tokens": 23086742
+    },
+    {
+      "epoch": 0.881470367591898,
+      "grad_norm": 0.19112971425056458,
+      "learning_rate": 0.00016478523965399085,
+      "loss": 0.5434,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7219,
+      "step": 1175,
+      "tokens_per_second_per_gpu": 4205.37,
+      "total_tokens": 23528106
+    },
+    {
+      "epoch": 0.900225056264066,
+      "grad_norm": 0.17930163443088531,
+      "learning_rate": 0.00016323652935929536,
+      "loss": 0.5362,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7095,
+      "step": 1200,
+      "tokens_per_second_per_gpu": 4228.83,
+      "total_tokens": 23974427
+    },
+    {
+      "epoch": 0.918979744936234,
+      "grad_norm": 0.18718039989471436,
+      "learning_rate": 0.00016166212130956382,
+      "loss": 0.5533,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.739,
+      "step": 1225,
+      "tokens_per_second_per_gpu": 4211.64,
+      "total_tokens": 24415919
+    },
+    {
+      "epoch": 0.9377344336084021,
+      "grad_norm": 0.17105573415756226,
+      "learning_rate": 0.0001600626553051268,
+      "loss": 0.5492,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7319,
+      "step": 1250,
+      "tokens_per_second_per_gpu": 4183.86,
+      "total_tokens": 24854345
+    },
+    {
+      "epoch": 0.9564891222805701,
+      "grad_norm": 0.1733955442905426,
+      "learning_rate": 0.0001584387813292454,
+      "loss": 0.5348,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7071,
+      "step": 1275,
+      "tokens_per_second_per_gpu": 4172.93,
+      "total_tokens": 25292647
+    },
+    {
+      "epoch": 0.9752438109527382,
+      "grad_norm": 0.1858205944299698,
+      "learning_rate": 0.00015679115928397401,
+      "loss": 0.5527,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7379,
+      "step": 1300,
+      "tokens_per_second_per_gpu": 4226.34,
+      "total_tokens": 25733591
+    },
+    {
+      "epoch": 0.9939984996249063,
+      "grad_norm": 0.1944192498922348,
+      "learning_rate": 0.00015512045872199276,
+      "loss": 0.5311,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7008,
+      "step": 1325,
+      "tokens_per_second_per_gpu": 3655.12,
+      "total_tokens": 26164528
+    },
+    {
+      "epoch": 1.0127531882970742,
+      "grad_norm": 0.18358173966407776,
+      "learning_rate": 0.00015342735857451777,
+      "loss": 0.5145,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6728,
+      "step": 1350,
+      "tokens_per_second_per_gpu": 4227.25,
+      "total_tokens": 26610460
+    },
+    {
+      "epoch": 1.0315078769692423,
+      "grad_norm": 0.1853465735912323,
+      "learning_rate": 0.00015171254687540038,
+      "loss": 0.5081,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6621,
+      "step": 1375,
+      "tokens_per_second_per_gpu": 4318.88,
+      "total_tokens": 27064008
+    },
+    {
+      "epoch": 1.0502625656414104,
+      "grad_norm": 0.18925060331821442,
+      "learning_rate": 0.0001499767204815273,
+      "loss": 0.5185,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6795,
+      "step": 1400,
+      "tokens_per_second_per_gpu": 4324.01,
+      "total_tokens": 27516590
+    },
+    {
+      "epoch": 1.0690172543135783,
+      "grad_norm": 0.20961470901966095,
+      "learning_rate": 0.00014822058478963532,
+      "loss": 0.5234,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6878,
+      "step": 1425,
+      "tokens_per_second_per_gpu": 4319.64,
+      "total_tokens": 27970075
+    },
+    {
+      "epoch": 1.0877719429857464,
+      "grad_norm": 0.1982697695493698,
+      "learning_rate": 0.0001464448534496555,
+      "loss": 0.5169,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6768,
+      "step": 1450,
+      "tokens_per_second_per_gpu": 4267.88,
+      "total_tokens": 28419716
+    },
+    {
+      "epoch": 1.1065266316579145,
+      "grad_norm": 0.1925143301486969,
+      "learning_rate": 0.00014465024807470376,
+      "loss": 0.5197,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6815,
+      "step": 1475,
+      "tokens_per_second_per_gpu": 4264.53,
+      "total_tokens": 28866312
+    },
+    {
+      "epoch": 1.1252813203300824,
+      "grad_norm": 0.18788637220859528,
+      "learning_rate": 0.0001428374979478349,
+      "loss": 0.5204,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6827,
+      "step": 1500,
+      "tokens_per_second_per_gpu": 3779.33,
+      "total_tokens": 29315968
+    },
+    {
+      "epoch": 1.1440360090022506,
+      "grad_norm": 0.18954145908355713,
+      "learning_rate": 0.00014100733972568038,
+      "loss": 0.5164,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.676,
+      "step": 1525,
+      "tokens_per_second_per_gpu": 4282.57,
+      "total_tokens": 29766723
+    },
+    {
+      "epoch": 1.1627906976744187,
+      "grad_norm": 0.19003146886825562,
+      "learning_rate": 0.00013916051713908924,
+      "loss": 0.5095,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6645,
+      "step": 1550,
+      "tokens_per_second_per_gpu": 4290.76,
+      "total_tokens": 30218573
+    },
+    {
+      "epoch": 1.1815453863465866,
+      "grad_norm": 0.18279583752155304,
+      "learning_rate": 0.00013729778069089437,
+      "loss": 0.522,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6854,
+      "step": 1575,
+      "tokens_per_second_per_gpu": 4300.13,
+      "total_tokens": 30669810
+    },
+    {
+      "epoch": 1.2003000750187547,
+      "grad_norm": 0.18783092498779297,
+      "learning_rate": 0.00013541988735092672,
+      "loss": 0.5003,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6492,
+      "step": 1600,
+      "tokens_per_second_per_gpu": 4271.27,
+      "total_tokens": 31117586
+    },
+    {
+      "epoch": 1.2190547636909228,
+      "grad_norm": 0.199558824300766,
+      "learning_rate": 0.00013352760024840175,
+      "loss": 0.5115,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6678,
+      "step": 1625,
+      "tokens_per_second_per_gpu": 4248.14,
+      "total_tokens": 31562224
+    },
+    {
+      "epoch": 1.2378094523630907,
+      "grad_norm": 0.19465653598308563,
+      "learning_rate": 0.00013162168836180246,
+      "loss": 0.4967,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6433,
+      "step": 1650,
+      "tokens_per_second_per_gpu": 4286.24,
+      "total_tokens": 32011071
+    },
+    {
+      "epoch": 1.2565641410352588,
+      "grad_norm": 0.2054641842842102,
+      "learning_rate": 0.00012970292620638574,
+      "loss": 0.5172,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6773,
+      "step": 1675,
+      "tokens_per_second_per_gpu": 3733.1,
+      "total_tokens": 32452490
+    },
+    {
+      "epoch": 1.275318829707427,
+      "grad_norm": 0.19450411200523376,
+      "learning_rate": 0.00012777209351943862,
+      "loss": 0.5149,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6735,
+      "step": 1700,
+      "tokens_per_second_per_gpu": 4251.33,
+      "total_tokens": 32899103
+    },
+    {
+      "epoch": 1.2940735183795948,
+      "grad_norm": 0.19844166934490204,
+      "learning_rate": 0.0001258299749434123,
+      "loss": 0.5205,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6829,
+      "step": 1725,
+      "tokens_per_second_per_gpu": 4240.57,
+      "total_tokens": 33344569
+    },
+    {
+      "epoch": 1.312828207051763,
+      "grad_norm": 0.19240470230579376,
+      "learning_rate": 0.00012387735970706312,
+      "loss": 0.5033,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6542,
+      "step": 1750,
+      "tokens_per_second_per_gpu": 4267.65,
+      "total_tokens": 33790426
+    },
+    {
+      "epoch": 1.331582895723931,
+      "grad_norm": 0.18220192193984985,
+      "learning_rate": 0.00012191504130472937,
+      "loss": 0.5103,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6658,
+      "step": 1775,
+      "tokens_per_second_per_gpu": 4237.08,
+      "total_tokens": 34233908
+    },
+    {
+      "epoch": 1.350337584396099,
+      "grad_norm": 0.20157551765441895,
+      "learning_rate": 0.00011994381717387514,
+      "loss": 0.5192,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6807,
+      "step": 1800,
+      "tokens_per_second_per_gpu": 4244.09,
+      "total_tokens": 34678691
+    },
+    {
+      "epoch": 1.369092273068267,
+      "grad_norm": 0.17189238965511322,
+      "learning_rate": 0.00011796448837103129,
+      "loss": 0.5011,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6505,
+      "step": 1825,
+      "tokens_per_second_per_gpu": 4277.26,
+      "total_tokens": 35125624
+    },
+    {
+      "epoch": 1.387846961740435,
+      "grad_norm": 0.19443106651306152,
+      "learning_rate": 0.00011597785924626616,
+      "loss": 0.4994,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6477,
+      "step": 1850,
+      "tokens_per_second_per_gpu": 3766.52,
+      "total_tokens": 35568850
+    },
+    {
+      "epoch": 1.406601650412603,
+      "grad_norm": 0.1810811311006546,
+      "learning_rate": 0.00011398473711631764,
+      "loss": 0.5083,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6625,
+      "step": 1875,
+      "tokens_per_second_per_gpu": 4204.76,
+      "total_tokens": 36009980
+    },
+    {
+      "epoch": 1.4253563390847712,
+      "grad_norm": 0.19805970788002014,
+      "learning_rate": 0.00011198593193651958,
+      "loss": 0.5141,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6721,
+      "step": 1900,
+      "tokens_per_second_per_gpu": 4270.21,
+      "total_tokens": 36457032
+    },
+    {
+      "epoch": 1.4441110277569393,
+      "grad_norm": 0.1936168372631073,
+      "learning_rate": 0.00010998225597165628,
+      "loss": 0.5045,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6562,
+      "step": 1925,
+      "tokens_per_second_per_gpu": 4275.24,
+      "total_tokens": 36905590
+    },
+    {
+      "epoch": 1.4628657164291072,
+      "grad_norm": 0.19065748155117035,
+      "learning_rate": 0.00010797452346587798,
+      "loss": 0.5025,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6528,
+      "step": 1950,
+      "tokens_per_second_per_gpu": 4285.81,
+      "total_tokens": 37354436
+    },
+    {
+      "epoch": 1.4816204051012754,
+      "grad_norm": 0.18647657334804535,
+      "learning_rate": 0.0001059635503118125,
+      "loss": 0.5102,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6656,
+      "step": 1975,
+      "tokens_per_second_per_gpu": 4259.76,
+      "total_tokens": 37801500
+    },
+    {
+      "epoch": 1.5003750937734432,
+      "grad_norm": 0.21211788058280945,
+      "learning_rate": 0.00010395015371900663,
+      "loss": 0.5052,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6573,
+      "step": 2000,
+      "tokens_per_second_per_gpu": 4250.7,
+      "total_tokens": 38244936
+    },
+    {
+      "epoch": 1.5003750937734432,
+      "eval_loss": 0.5063687562942505,
+      "eval_ppl": 1.6593,
+      "eval_runtime": 141.112,
+      "eval_samples_per_second": 5.173,
+      "eval_steps_per_second": 1.035,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 19.1,
+      "memory/max_allocated (GiB)": 19.1,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 3996,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.62796004179968e+18,
+  "train_batch_size": 5,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a75cafa9afe18e1aa48e766453f03add03ceb5dd78011703e6068968ae04eab2
+size 7761

checkpoint-3000/README.md

@@ -0,0 +1,208 @@
+---
+base_model: codellama/CodeLlama-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:codellama/CodeLlama-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.0

checkpoint-3000/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "codellama/CodeLlama-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-3000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f5f1ada7c250db48cba2e238791cdb7014809135a500d875060d02a931d53e
+size 67143296

checkpoint-3000/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-3000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20304343455a8fd488379728c67bf039255771ac30db5a1d3a9260acecb9f9b8
+size 134433995

checkpoint-3000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5a2fdb47ec839a29b69def75adfef309f87d753c90579b6aaac5f1d4be8e923
+size 14645

checkpoint-3000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c0d3d381187345aa455213a26a4e087c224775b41b6b4866e6c4ddd1abcdf5f
+size 1465

checkpoint-3000/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-3000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ccb9c8b6b561889acea59191d66986d314e7cbd6a78abc6e49b139ca91c1e6
+size 500058

checkpoint-3000/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

checkpoint-3000/trainer_state.json

@@ -0,0 +1,1642 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.250562640660165,
+  "eval_steps": 1000,
+  "global_step": 3000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0,
+      "eval_loss": 1.6887853145599365,
+      "eval_ppl": 5.4129,
+      "eval_runtime": 167.3526,
+      "eval_samples_per_second": 4.362,
+      "eval_steps_per_second": 0.872,
+      "memory/device_reserved (GiB)": 139.12,
+      "memory/max_active (GiB)": 18.94,
+      "memory/max_allocated (GiB)": 18.94,
+      "step": 0
+    },
+    {
+      "epoch": 0.018754688672168042,
+      "grad_norm": 1.415561556816101,
+      "learning_rate": 4.8e-05,
+      "loss": 1.6848,
+      "memory/device_reserved (GiB)": 139.11,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 5.3914,
+      "step": 25,
+      "tokens_per_second_per_gpu": 16277.76,
+      "total_tokens": 1723633
+    },
+    {
+      "epoch": 0.037509377344336084,
+      "grad_norm": 0.33179354667663574,
+      "learning_rate": 9.8e-05,
+      "loss": 0.9839,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.6749,
+      "step": 50,
+      "tokens_per_second_per_gpu": 4303.21,
+      "total_tokens": 2175386
+    },
+    {
+      "epoch": 0.056264066016504126,
+      "grad_norm": 0.17453454434871674,
+      "learning_rate": 0.000148,
+      "loss": 0.8002,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.226,
+      "step": 75,
+      "tokens_per_second_per_gpu": 3776.03,
+      "total_tokens": 2623712
+    },
+    {
+      "epoch": 0.07501875468867217,
+      "grad_norm": 0.19318008422851562,
+      "learning_rate": 0.00019800000000000002,
+      "loss": 0.7218,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.0581,
+      "step": 100,
+      "tokens_per_second_per_gpu": 4252.49,
+      "total_tokens": 3072519
+    },
+    {
+      "epoch": 0.09377344336084022,
+      "grad_norm": 0.18435686826705933,
+      "learning_rate": 0.00019998127418269004,
+      "loss": 0.6759,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.9658,
+      "step": 125,
+      "tokens_per_second_per_gpu": 4303.31,
+      "total_tokens": 3523983
+    },
+    {
+      "epoch": 0.11252813203300825,
+      "grad_norm": 0.19870473444461823,
+      "learning_rate": 0.00019992195096972548,
+      "loss": 0.6703,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.9548,
+      "step": 150,
+      "tokens_per_second_per_gpu": 4260.86,
+      "total_tokens": 3973452
+    },
+    {
+      "epoch": 0.1312828207051763,
+      "grad_norm": 0.20499658584594727,
+      "learning_rate": 0.0001998220219574743,
+      "loss": 0.6381,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8929,
+      "step": 175,
+      "tokens_per_second_per_gpu": 4288.64,
+      "total_tokens": 4423763
+    },
+    {
+      "epoch": 0.15003750937734434,
+      "grad_norm": 0.18934418261051178,
+      "learning_rate": 0.00019968152775460537,
+      "loss": 0.6383,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8933,
+      "step": 200,
+      "tokens_per_second_per_gpu": 4244.79,
+      "total_tokens": 4872365
+    },
+    {
+      "epoch": 0.16879219804951237,
+      "grad_norm": 0.1827855408191681,
+      "learning_rate": 0.00019950052545447352,
+      "loss": 0.6347,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8865,
+      "step": 225,
+      "tokens_per_second_per_gpu": 4252.71,
+      "total_tokens": 5319322
+    },
+    {
+      "epoch": 0.18754688672168043,
+      "grad_norm": 0.16483066976070404,
+      "learning_rate": 0.00019927908861191827,
+      "loss": 0.6392,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.895,
+      "step": 250,
+      "tokens_per_second_per_gpu": 3772.0,
+      "total_tokens": 5768644
+    },
+    {
+      "epoch": 0.20630157539384847,
+      "grad_norm": 0.17186357080936432,
+      "learning_rate": 0.00019901730721337302,
+      "loss": 0.614,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8478,
+      "step": 275,
+      "tokens_per_second_per_gpu": 4281.82,
+      "total_tokens": 6220751
+    },
+    {
+      "epoch": 0.2250562640660165,
+      "grad_norm": 0.18073013424873352,
+      "learning_rate": 0.00019871528764029667,
+      "loss": 0.6196,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8582,
+      "step": 300,
+      "tokens_per_second_per_gpu": 4234.51,
+      "total_tokens": 6668111
+    },
+    {
+      "epoch": 0.24381095273818454,
+      "grad_norm": 0.19639697670936584,
+      "learning_rate": 0.00019837315262594306,
+      "loss": 0.6181,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8554,
+      "step": 325,
+      "tokens_per_second_per_gpu": 4261.44,
+      "total_tokens": 7117439
+    },
+    {
+      "epoch": 0.2625656414103526,
+      "grad_norm": 0.1670486479997635,
+      "learning_rate": 0.00019799104120548492,
+      "loss": 0.6141,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.848,
+      "step": 350,
+      "tokens_per_second_per_gpu": 4298.97,
+      "total_tokens": 7569060
+    },
+    {
+      "epoch": 0.2813203300825206,
+      "grad_norm": 0.17752495408058167,
+      "learning_rate": 0.00019756910865951377,
+      "loss": 0.6075,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8358,
+      "step": 375,
+      "tokens_per_second_per_gpu": 4256.6,
+      "total_tokens": 8017630
+    },
+    {
+      "epoch": 0.30007501875468867,
+      "grad_norm": 0.2000180035829544,
+      "learning_rate": 0.00019710752645093747,
+      "loss": 0.6108,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8419,
+      "step": 400,
+      "tokens_per_second_per_gpu": 4245.46,
+      "total_tokens": 8464998
+    },
+    {
+      "epoch": 0.31882970742685673,
+      "grad_norm": 0.17395919561386108,
+      "learning_rate": 0.00019660648215530206,
+      "loss": 0.5966,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8159,
+      "step": 425,
+      "tokens_per_second_per_gpu": 3758.92,
+      "total_tokens": 8914723
+    },
+    {
+      "epoch": 0.33758439609902474,
+      "grad_norm": 0.18785236775875092,
+      "learning_rate": 0.00019606617938456572,
+      "loss": 0.6099,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8402,
+      "step": 450,
+      "tokens_per_second_per_gpu": 4200.53,
+      "total_tokens": 9359638
+    },
+    {
+      "epoch": 0.3563390847711928,
+      "grad_norm": 0.17702797055244446,
+      "learning_rate": 0.0001954868377043559,
+      "loss": 0.5922,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.808,
+      "step": 475,
+      "tokens_per_second_per_gpu": 4265.36,
+      "total_tokens": 9810837
+    },
+    {
+      "epoch": 0.37509377344336087,
+      "grad_norm": 0.19927558302879333,
+      "learning_rate": 0.00019486869254474337,
+      "loss": 0.5759,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7787,
+      "step": 500,
+      "tokens_per_second_per_gpu": 4276.25,
+      "total_tokens": 10261446
+    },
+    {
+      "epoch": 0.3938484621155289,
+      "grad_norm": 0.1908370852470398,
+      "learning_rate": 0.0001942119951045692,
+      "loss": 0.584,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7932,
+      "step": 525,
+      "tokens_per_second_per_gpu": 4272.28,
+      "total_tokens": 10707841
+    },
+    {
+      "epoch": 0.41260315078769694,
+      "grad_norm": 0.2064146101474762,
+      "learning_rate": 0.00019351701224936383,
+      "loss": 0.5791,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7844,
+      "step": 550,
+      "tokens_per_second_per_gpu": 4250.37,
+      "total_tokens": 11155384
+    },
+    {
+      "epoch": 0.43135783945986494,
+      "grad_norm": 0.26748332381248474,
+      "learning_rate": 0.0001927840264028995,
+      "loss": 0.5758,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7786,
+      "step": 575,
+      "tokens_per_second_per_gpu": 4256.55,
+      "total_tokens": 11601192
+    },
+    {
+      "epoch": 0.450112528132033,
+      "grad_norm": 0.17514832317829132,
+      "learning_rate": 0.00019201333543242036,
+      "loss": 0.5791,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7844,
+      "step": 600,
+      "tokens_per_second_per_gpu": 3770.83,
+      "total_tokens": 12048477
+    },
+    {
+      "epoch": 0.46886721680420107,
+      "grad_norm": 0.22069169580936432,
+      "learning_rate": 0.00019120525252759647,
+      "loss": 0.5803,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7866,
+      "step": 625,
+      "tokens_per_second_per_gpu": 4179.31,
+      "total_tokens": 12488141
+    },
+    {
+      "epoch": 0.4876219054763691,
+      "grad_norm": 0.20555566251277924,
+      "learning_rate": 0.00019036010607325138,
+      "loss": 0.5716,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7711,
+      "step": 650,
+      "tokens_per_second_per_gpu": 4209.96,
+      "total_tokens": 12934358
+    },
+    {
+      "epoch": 0.5063765941485371,
+      "grad_norm": 0.19018156826496124,
+      "learning_rate": 0.00018947823951591478,
+      "loss": 0.5608,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7521,
+      "step": 675,
+      "tokens_per_second_per_gpu": 4226.4,
+      "total_tokens": 13378983
+    },
+    {
+      "epoch": 0.5251312828207052,
+      "grad_norm": 0.17173859477043152,
+      "learning_rate": 0.00018856001122425416,
+      "loss": 0.5667,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7624,
+      "step": 700,
+      "tokens_per_second_per_gpu": 4265.57,
+      "total_tokens": 13829519
+    },
+    {
+      "epoch": 0.5438859714928732,
+      "grad_norm": 0.17706550657749176,
+      "learning_rate": 0.0001876057943434428,
+      "loss": 0.565,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7594,
+      "step": 725,
+      "tokens_per_second_per_gpu": 4281.61,
+      "total_tokens": 14281879
+    },
+    {
+      "epoch": 0.5626406601650412,
+      "grad_norm": 0.18528586626052856,
+      "learning_rate": 0.00018661597664352284,
+      "loss": 0.5666,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7623,
+      "step": 750,
+      "tokens_per_second_per_gpu": 4229.32,
+      "total_tokens": 14725919
+    },
+    {
+      "epoch": 0.5813953488372093,
+      "grad_norm": 0.16790929436683655,
+      "learning_rate": 0.00018559096036182516,
+      "loss": 0.5633,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7565,
+      "step": 775,
+      "tokens_per_second_per_gpu": 3775.0,
+      "total_tokens": 15175146
+    },
+    {
+      "epoch": 0.6001500375093773,
+      "grad_norm": 0.17511805891990662,
+      "learning_rate": 0.00018453116203951005,
+      "loss": 0.5664,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7619,
+      "step": 800,
+      "tokens_per_second_per_gpu": 4218.07,
+      "total_tokens": 15619901
+    },
+    {
+      "epoch": 0.6189047261815454,
+      "grad_norm": 0.19853387773036957,
+      "learning_rate": 0.0001834370123522954,
+      "loss": 0.5646,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7587,
+      "step": 825,
+      "tokens_per_second_per_gpu": 4230.84,
+      "total_tokens": 16066102
+    },
+    {
+      "epoch": 0.6376594148537135,
+      "grad_norm": 0.18872258067131042,
+      "learning_rate": 0.00018230895593544056,
+      "loss": 0.552,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7367,
+      "step": 850,
+      "tokens_per_second_per_gpu": 4222.33,
+      "total_tokens": 16510696
+    },
+    {
+      "epoch": 0.6564141035258815,
+      "grad_norm": 0.9702818989753723,
+      "learning_rate": 0.0001811474512030578,
+      "loss": 0.5607,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7519,
+      "step": 875,
+      "tokens_per_second_per_gpu": 4200.39,
+      "total_tokens": 16953918
+    },
+    {
+      "epoch": 0.6751687921980495,
+      "grad_norm": 0.17479568719863892,
+      "learning_rate": 0.00017995297016182405,
+      "loss": 0.564,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7577,
+      "step": 900,
+      "tokens_per_second_per_gpu": 4210.15,
+      "total_tokens": 17396453
+    },
+    {
+      "epoch": 0.6939234808702176,
+      "grad_norm": 0.1948954463005066,
+      "learning_rate": 0.0001787259982191692,
+      "loss": 0.5511,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7352,
+      "step": 925,
+      "tokens_per_second_per_gpu": 4237.98,
+      "total_tokens": 17841287
+    },
+    {
+      "epoch": 0.7126781695423856,
+      "grad_norm": 0.19541053473949432,
+      "learning_rate": 0.00017746703398601872,
+      "loss": 0.5532,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7388,
+      "step": 950,
+      "tokens_per_second_per_gpu": 3725.33,
+      "total_tokens": 18283596
+    },
+    {
+      "epoch": 0.7314328582145536,
+      "grad_norm": 0.1818365603685379,
+      "learning_rate": 0.0001761765890741701,
+      "loss": 0.5521,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7369,
+      "step": 975,
+      "tokens_per_second_per_gpu": 4211.63,
+      "total_tokens": 18726722
+    },
+    {
+      "epoch": 0.7501875468867217,
+      "grad_norm": 0.1838025599718094,
+      "learning_rate": 0.00017485518788838705,
+      "loss": 0.5511,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7352,
+      "step": 1000,
+      "tokens_per_second_per_gpu": 3962.4,
+      "total_tokens": 19167258
+    },
+    {
+      "epoch": 0.7501875468867217,
+      "eval_loss": 0.540988564491272,
+      "eval_ppl": 1.7177,
+      "eval_runtime": 138.0264,
+      "eval_samples_per_second": 5.289,
+      "eval_steps_per_second": 1.058,
+      "memory/device_reserved (GiB)": 139.02,
+      "memory/max_active (GiB)": 19.1,
+      "memory/max_allocated (GiB)": 19.1,
+      "step": 1000
+    },
+    {
+      "epoch": 0.7689422355588897,
+      "grad_norm": 0.2199818342924118,
+      "learning_rate": 0.00017350336741329413,
+      "loss": 0.549,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7315,
+      "step": 1025,
+      "tokens_per_second_per_gpu": 4129.73,
+      "total_tokens": 20870820
+    },
+    {
+      "epoch": 0.7876969242310577,
+      "grad_norm": 0.19783177971839905,
+      "learning_rate": 0.0001721216769951596,
+      "loss": 0.5615,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7533,
+      "step": 1050,
+      "tokens_per_second_per_gpu": 4243.63,
+      "total_tokens": 21317982
+    },
+    {
+      "epoch": 0.8064516129032258,
+      "grad_norm": 0.1678430140018463,
+      "learning_rate": 0.00017071067811865476,
+      "loss": 0.5557,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7432,
+      "step": 1075,
+      "tokens_per_second_per_gpu": 4092.04,
+      "total_tokens": 21754087
+    },
+    {
+      "epoch": 0.8252063015753939,
+      "grad_norm": 0.16523879766464233,
+      "learning_rate": 0.00016927094417868048,
+      "loss": 0.556,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7437,
+      "step": 1100,
+      "tokens_per_second_per_gpu": 4187.02,
+      "total_tokens": 22198779
+    },
+    {
+      "epoch": 0.8439609902475619,
+      "grad_norm": 0.18177717924118042,
+      "learning_rate": 0.00016780306024735382,
+      "loss": 0.5468,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7277,
+      "step": 1125,
+      "tokens_per_second_per_gpu": 4198.97,
+      "total_tokens": 22639769
+    },
+    {
+      "epoch": 0.8627156789197299,
+      "grad_norm": 0.17299720644950867,
+      "learning_rate": 0.0001663076228362492,
+      "loss": 0.554,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7402,
+      "step": 1150,
+      "tokens_per_second_per_gpu": 3762.13,
+      "total_tokens": 23086742
+    },
+    {
+      "epoch": 0.881470367591898,
+      "grad_norm": 0.19112971425056458,
+      "learning_rate": 0.00016478523965399085,
+      "loss": 0.5434,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7219,
+      "step": 1175,
+      "tokens_per_second_per_gpu": 4205.37,
+      "total_tokens": 23528106
+    },
+    {
+      "epoch": 0.900225056264066,
+      "grad_norm": 0.17930163443088531,
+      "learning_rate": 0.00016323652935929536,
+      "loss": 0.5362,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7095,
+      "step": 1200,
+      "tokens_per_second_per_gpu": 4228.83,
+      "total_tokens": 23974427
+    },
+    {
+      "epoch": 0.918979744936234,
+      "grad_norm": 0.18718039989471436,
+      "learning_rate": 0.00016166212130956382,
+      "loss": 0.5533,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.739,
+      "step": 1225,
+      "tokens_per_second_per_gpu": 4211.64,
+      "total_tokens": 24415919
+    },
+    {
+      "epoch": 0.9377344336084021,
+      "grad_norm": 0.17105573415756226,
+      "learning_rate": 0.0001600626553051268,
+      "loss": 0.5492,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7319,
+      "step": 1250,
+      "tokens_per_second_per_gpu": 4183.86,
+      "total_tokens": 24854345
+    },
+    {
+      "epoch": 0.9564891222805701,
+      "grad_norm": 0.1733955442905426,
+      "learning_rate": 0.0001584387813292454,
+      "loss": 0.5348,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7071,
+      "step": 1275,
+      "tokens_per_second_per_gpu": 4172.93,
+      "total_tokens": 25292647
+    },
+    {
+      "epoch": 0.9752438109527382,
+      "grad_norm": 0.1858205944299698,
+      "learning_rate": 0.00015679115928397401,
+      "loss": 0.5527,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7379,
+      "step": 1300,
+      "tokens_per_second_per_gpu": 4226.34,
+      "total_tokens": 25733591
+    },
+    {
+      "epoch": 0.9939984996249063,
+      "grad_norm": 0.1944192498922348,
+      "learning_rate": 0.00015512045872199276,
+      "loss": 0.5311,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7008,
+      "step": 1325,
+      "tokens_per_second_per_gpu": 3655.12,
+      "total_tokens": 26164528
+    },
+    {
+      "epoch": 1.0127531882970742,
+      "grad_norm": 0.18358173966407776,
+      "learning_rate": 0.00015342735857451777,
+      "loss": 0.5145,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6728,
+      "step": 1350,
+      "tokens_per_second_per_gpu": 4227.25,
+      "total_tokens": 26610460
+    },
+    {
+      "epoch": 1.0315078769692423,
+      "grad_norm": 0.1853465735912323,
+      "learning_rate": 0.00015171254687540038,
+      "loss": 0.5081,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6621,
+      "step": 1375,
+      "tokens_per_second_per_gpu": 4318.88,
+      "total_tokens": 27064008
+    },
+    {
+      "epoch": 1.0502625656414104,
+      "grad_norm": 0.18925060331821442,
+      "learning_rate": 0.0001499767204815273,
+      "loss": 0.5185,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6795,
+      "step": 1400,
+      "tokens_per_second_per_gpu": 4324.01,
+      "total_tokens": 27516590
+    },
+    {
+      "epoch": 1.0690172543135783,
+      "grad_norm": 0.20961470901966095,
+      "learning_rate": 0.00014822058478963532,
+      "loss": 0.5234,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6878,
+      "step": 1425,
+      "tokens_per_second_per_gpu": 4319.64,
+      "total_tokens": 27970075
+    },
+    {
+      "epoch": 1.0877719429857464,
+      "grad_norm": 0.1982697695493698,
+      "learning_rate": 0.0001464448534496555,
+      "loss": 0.5169,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6768,
+      "step": 1450,
+      "tokens_per_second_per_gpu": 4267.88,
+      "total_tokens": 28419716
+    },
+    {
+      "epoch": 1.1065266316579145,
+      "grad_norm": 0.1925143301486969,
+      "learning_rate": 0.00014465024807470376,
+      "loss": 0.5197,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6815,
+      "step": 1475,
+      "tokens_per_second_per_gpu": 4264.53,
+      "total_tokens": 28866312
+    },
+    {
+      "epoch": 1.1252813203300824,
+      "grad_norm": 0.18788637220859528,
+      "learning_rate": 0.0001428374979478349,
+      "loss": 0.5204,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6827,
+      "step": 1500,
+      "tokens_per_second_per_gpu": 3779.33,
+      "total_tokens": 29315968
+    },
+    {
+      "epoch": 1.1440360090022506,
+      "grad_norm": 0.18954145908355713,
+      "learning_rate": 0.00014100733972568038,
+      "loss": 0.5164,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.676,
+      "step": 1525,
+      "tokens_per_second_per_gpu": 4282.57,
+      "total_tokens": 29766723
+    },
+    {
+      "epoch": 1.1627906976744187,
+      "grad_norm": 0.19003146886825562,
+      "learning_rate": 0.00013916051713908924,
+      "loss": 0.5095,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6645,
+      "step": 1550,
+      "tokens_per_second_per_gpu": 4290.76,
+      "total_tokens": 30218573
+    },
+    {
+      "epoch": 1.1815453863465866,
+      "grad_norm": 0.18279583752155304,
+      "learning_rate": 0.00013729778069089437,
+      "loss": 0.522,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6854,
+      "step": 1575,
+      "tokens_per_second_per_gpu": 4300.13,
+      "total_tokens": 30669810
+    },
+    {
+      "epoch": 1.2003000750187547,
+      "grad_norm": 0.18783092498779297,
+      "learning_rate": 0.00013541988735092672,
+      "loss": 0.5003,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6492,
+      "step": 1600,
+      "tokens_per_second_per_gpu": 4271.27,
+      "total_tokens": 31117586
+    },
+    {
+      "epoch": 1.2190547636909228,
+      "grad_norm": 0.199558824300766,
+      "learning_rate": 0.00013352760024840175,
+      "loss": 0.5115,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6678,
+      "step": 1625,
+      "tokens_per_second_per_gpu": 4248.14,
+      "total_tokens": 31562224
+    },
+    {
+      "epoch": 1.2378094523630907,
+      "grad_norm": 0.19465653598308563,
+      "learning_rate": 0.00013162168836180246,
+      "loss": 0.4967,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6433,
+      "step": 1650,
+      "tokens_per_second_per_gpu": 4286.24,
+      "total_tokens": 32011071
+    },
+    {
+      "epoch": 1.2565641410352588,
+      "grad_norm": 0.2054641842842102,
+      "learning_rate": 0.00012970292620638574,
+      "loss": 0.5172,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6773,
+      "step": 1675,
+      "tokens_per_second_per_gpu": 3733.1,
+      "total_tokens": 32452490
+    },
+    {
+      "epoch": 1.275318829707427,
+      "grad_norm": 0.19450411200523376,
+      "learning_rate": 0.00012777209351943862,
+      "loss": 0.5149,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6735,
+      "step": 1700,
+      "tokens_per_second_per_gpu": 4251.33,
+      "total_tokens": 32899103
+    },
+    {
+      "epoch": 1.2940735183795948,
+      "grad_norm": 0.19844166934490204,
+      "learning_rate": 0.0001258299749434123,
+      "loss": 0.5205,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6829,
+      "step": 1725,
+      "tokens_per_second_per_gpu": 4240.57,
+      "total_tokens": 33344569
+    },
+    {
+      "epoch": 1.312828207051763,
+      "grad_norm": 0.19240470230579376,
+      "learning_rate": 0.00012387735970706312,
+      "loss": 0.5033,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6542,
+      "step": 1750,
+      "tokens_per_second_per_gpu": 4267.65,
+      "total_tokens": 33790426
+    },
+    {
+      "epoch": 1.331582895723931,
+      "grad_norm": 0.18220192193984985,
+      "learning_rate": 0.00012191504130472937,
+      "loss": 0.5103,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6658,
+      "step": 1775,
+      "tokens_per_second_per_gpu": 4237.08,
+      "total_tokens": 34233908
+    },
+    {
+      "epoch": 1.350337584396099,
+      "grad_norm": 0.20157551765441895,
+      "learning_rate": 0.00011994381717387514,
+      "loss": 0.5192,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6807,
+      "step": 1800,
+      "tokens_per_second_per_gpu": 4244.09,
+      "total_tokens": 34678691
+    },
+    {
+      "epoch": 1.369092273068267,
+      "grad_norm": 0.17189238965511322,
+      "learning_rate": 0.00011796448837103129,
+      "loss": 0.5011,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6505,
+      "step": 1825,
+      "tokens_per_second_per_gpu": 4277.26,
+      "total_tokens": 35125624
+    },
+    {
+      "epoch": 1.387846961740435,
+      "grad_norm": 0.19443106651306152,
+      "learning_rate": 0.00011597785924626616,
+      "loss": 0.4994,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6477,
+      "step": 1850,
+      "tokens_per_second_per_gpu": 3766.52,
+      "total_tokens": 35568850
+    },
+    {
+      "epoch": 1.406601650412603,
+      "grad_norm": 0.1810811311006546,
+      "learning_rate": 0.00011398473711631764,
+      "loss": 0.5083,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6625,
+      "step": 1875,
+      "tokens_per_second_per_gpu": 4204.76,
+      "total_tokens": 36009980
+    },
+    {
+      "epoch": 1.4253563390847712,
+      "grad_norm": 0.19805970788002014,
+      "learning_rate": 0.00011198593193651958,
+      "loss": 0.5141,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6721,
+      "step": 1900,
+      "tokens_per_second_per_gpu": 4270.21,
+      "total_tokens": 36457032
+    },
+    {
+      "epoch": 1.4441110277569393,
+      "grad_norm": 0.1936168372631073,
+      "learning_rate": 0.00010998225597165628,
+      "loss": 0.5045,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6562,
+      "step": 1925,
+      "tokens_per_second_per_gpu": 4275.24,
+      "total_tokens": 36905590
+    },
+    {
+      "epoch": 1.4628657164291072,
+      "grad_norm": 0.19065748155117035,
+      "learning_rate": 0.00010797452346587798,
+      "loss": 0.5025,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6528,
+      "step": 1950,
+      "tokens_per_second_per_gpu": 4285.81,
+      "total_tokens": 37354436
+    },
+    {
+      "epoch": 1.4816204051012754,
+      "grad_norm": 0.18647657334804535,
+      "learning_rate": 0.0001059635503118125,
+      "loss": 0.5102,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6656,
+      "step": 1975,
+      "tokens_per_second_per_gpu": 4259.76,
+      "total_tokens": 37801500
+    },
+    {
+      "epoch": 1.5003750937734432,
+      "grad_norm": 0.21211788058280945,
+      "learning_rate": 0.00010395015371900663,
+      "loss": 0.5052,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6573,
+      "step": 2000,
+      "tokens_per_second_per_gpu": 4250.7,
+      "total_tokens": 38244936
+    },
+    {
+      "epoch": 1.5003750937734432,
+      "eval_loss": 0.5063687562942505,
+      "eval_ppl": 1.6593,
+      "eval_runtime": 141.112,
+      "eval_samples_per_second": 5.173,
+      "eval_steps_per_second": 1.035,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 19.1,
+      "memory/max_allocated (GiB)": 19.1,
+      "step": 2000
+    },
+    {
+      "epoch": 1.5191297824456114,
+      "grad_norm": 0.20089760422706604,
+      "learning_rate": 0.00010193515188183245,
+      "loss": 0.4892,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.631,
+      "step": 2025,
+      "tokens_per_second_per_gpu": 4246.58,
+      "total_tokens": 39959888
+    },
+    {
+      "epoch": 1.5378844711177795,
+      "grad_norm": 0.19840118288993835,
+      "learning_rate": 9.991936364699348e-05,
+      "loss": 0.503,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6537,
+      "step": 2050,
+      "tokens_per_second_per_gpu": 4320.38,
+      "total_tokens": 40411902
+    },
+    {
+      "epoch": 1.5566391597899476,
+      "grad_norm": 0.20045842230319977,
+      "learning_rate": 9.790360818076577e-05,
+      "loss": 0.5127,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6698,
+      "step": 2075,
+      "tokens_per_second_per_gpu": 4245.02,
+      "total_tokens": 40855384
+    },
+    {
+      "epoch": 1.5753938484621155,
+      "grad_norm": 0.19669026136398315,
+      "learning_rate": 9.588870463610893e-05,
+      "loss": 0.4994,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6477,
+      "step": 2100,
+      "tokens_per_second_per_gpu": 4174.18,
+      "total_tokens": 41293525
+    },
+    {
+      "epoch": 1.5941485371342836,
+      "grad_norm": 0.19754259288311005,
+      "learning_rate": 9.387547181978291e-05,
+      "loss": 0.5009,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6502,
+      "step": 2125,
+      "tokens_per_second_per_gpu": 4200.06,
+      "total_tokens": 41737747
+    },
+    {
+      "epoch": 1.6129032258064515,
+      "grad_norm": 0.19482502341270447,
+      "learning_rate": 9.186472785960507e-05,
+      "loss": 0.5002,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6491,
+      "step": 2150,
+      "tokens_per_second_per_gpu": 3696.76,
+      "total_tokens": 42176082
+    },
+    {
+      "epoch": 1.6316579144786196,
+      "grad_norm": 0.21606561541557312,
+      "learning_rate": 8.985728987198352e-05,
+      "loss": 0.4959,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.642,
+      "step": 2175,
+      "tokens_per_second_per_gpu": 4192.5,
+      "total_tokens": 42616372
+    },
+    {
+      "epoch": 1.6504126031507877,
+      "grad_norm": 0.1979638934135437,
+      "learning_rate": 8.785397362986114e-05,
+      "loss": 0.5031,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6538,
+      "step": 2200,
+      "tokens_per_second_per_gpu": 4211.67,
+      "total_tokens": 43058315
+    },
+    {
+      "epoch": 1.6691672918229559,
+      "grad_norm": 0.20717743039131165,
+      "learning_rate": 8.58555932312059e-05,
+      "loss": 0.4986,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6464,
+      "step": 2225,
+      "tokens_per_second_per_gpu": 4242.04,
+      "total_tokens": 43501960
+    },
+    {
+      "epoch": 1.6879219804951238,
+      "grad_norm": 0.18736609816551208,
+      "learning_rate": 8.38629607681815e-05,
+      "loss": 0.4898,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.632,
+      "step": 2250,
+      "tokens_per_second_per_gpu": 4235.21,
+      "total_tokens": 43947235
+    },
+    {
+      "epoch": 1.7066766691672917,
+      "grad_norm": 0.2056591659784317,
+      "learning_rate": 8.187688599713333e-05,
+      "loss": 0.4925,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6364,
+      "step": 2275,
+      "tokens_per_second_per_gpu": 4256.41,
+      "total_tokens": 44393451
+    },
+    {
+      "epoch": 1.7254313578394598,
+      "grad_norm": 0.19774597883224487,
+      "learning_rate": 7.989817600952376e-05,
+      "loss": 0.4952,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6408,
+      "step": 2300,
+      "tokens_per_second_per_gpu": 4224.5,
+      "total_tokens": 44836590
+    },
+    {
+      "epoch": 1.744186046511628,
+      "grad_norm": 0.19662383198738098,
+      "learning_rate": 7.792763490394984e-05,
+      "loss": 0.4977,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6449,
+      "step": 2325,
+      "tokens_per_second_per_gpu": 3741.52,
+      "total_tokens": 45279799
+    },
+    {
+      "epoch": 1.762940735183796,
+      "grad_norm": 0.19400179386138916,
+      "learning_rate": 7.596606345937812e-05,
+      "loss": 0.4965,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.643,
+      "step": 2350,
+      "tokens_per_second_per_gpu": 4248.51,
+      "total_tokens": 45725602
+    },
+    {
+      "epoch": 1.7816954238559641,
+      "grad_norm": 0.20261766016483307,
+      "learning_rate": 7.401425880972742e-05,
+      "loss": 0.5014,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.651,
+      "step": 2375,
+      "tokens_per_second_per_gpu": 4216.2,
+      "total_tokens": 46167730
+    },
+    {
+      "epoch": 1.800450112528132,
+      "grad_norm": 0.20447255671024323,
+      "learning_rate": 7.207301411993387e-05,
+      "loss": 0.4901,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6325,
+      "step": 2400,
+      "tokens_per_second_per_gpu": 3727.37,
+      "total_tokens": 46611126
+    },
+    {
+      "epoch": 1.8192048012003,
+      "grad_norm": 0.19921696186065674,
+      "learning_rate": 7.014311826362804e-05,
+      "loss": 0.4925,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6364,
+      "step": 2425,
+      "tokens_per_second_per_gpu": 4202.19,
+      "total_tokens": 47050763
+    },
+    {
+      "epoch": 1.837959489872468,
+      "grad_norm": 0.20095540583133698,
+      "learning_rate": 6.822535550255652e-05,
+      "loss": 0.494,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6389,
+      "step": 2450,
+      "tokens_per_second_per_gpu": 4230.16,
+      "total_tokens": 47496926
+    },
+    {
+      "epoch": 1.8567141785446362,
+      "grad_norm": 0.20210741460323334,
+      "learning_rate": 6.632050516787719e-05,
+      "loss": 0.5036,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6547,
+      "step": 2475,
+      "tokens_per_second_per_gpu": 4256.1,
+      "total_tokens": 47941250
+    },
+    {
+      "epoch": 1.8754688672168043,
+      "grad_norm": 0.21025419235229492,
+      "learning_rate": 6.442934134345871e-05,
+      "loss": 0.5019,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6519,
+      "step": 2500,
+      "tokens_per_second_per_gpu": 3728.09,
+      "total_tokens": 48383306
+    },
+    {
+      "epoch": 1.8942235558889724,
+      "grad_norm": 0.20130059123039246,
+      "learning_rate": 6.255263255131172e-05,
+      "loss": 0.5022,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6524,
+      "step": 2525,
+      "tokens_per_second_per_gpu": 4178.95,
+      "total_tokens": 48821862
+    },
+    {
+      "epoch": 1.9129782445611403,
+      "grad_norm": 0.19601669907569885,
+      "learning_rate": 6.0691141439280785e-05,
+      "loss": 0.4876,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6284,
+      "step": 2550,
+      "tokens_per_second_per_gpu": 3998.52,
+      "total_tokens": 49262344
+    },
+    {
+      "epoch": 1.9317329332333082,
+      "grad_norm": 0.20538586378097534,
+      "learning_rate": 5.884562447112331e-05,
+      "loss": 0.4796,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6154,
+      "step": 2575,
+      "tokens_per_second_per_gpu": 4192.8,
+      "total_tokens": 49702209
+    },
+    {
+      "epoch": 1.9504876219054763,
+      "grad_norm": 0.19957959651947021,
+      "learning_rate": 5.701683161910115e-05,
+      "loss": 0.5017,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6515,
+      "step": 2600,
+      "tokens_per_second_per_gpu": 4244.94,
+      "total_tokens": 50147673
+    },
+    {
+      "epoch": 1.9692423105776444,
+      "grad_norm": 0.20284536480903625,
+      "learning_rate": 5.520550605921091e-05,
+      "loss": 0.5024,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6527,
+      "step": 2625,
+      "tokens_per_second_per_gpu": 4205.45,
+      "total_tokens": 50589478
+    },
+    {
+      "epoch": 1.9879969992498125,
+      "grad_norm": 0.2044789344072342,
+      "learning_rate": 5.34123838691753e-05,
+      "loss": 0.4967,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6433,
+      "step": 2650,
+      "tokens_per_second_per_gpu": 4204.9,
+      "total_tokens": 51027800
+    },
+    {
+      "epoch": 2.0067516879219807,
+      "grad_norm": 0.2125943899154663,
+      "learning_rate": 5.163819372931979e-05,
+      "loss": 0.4862,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6261,
+      "step": 2675,
+      "tokens_per_second_per_gpu": 3745.54,
+      "total_tokens": 51469941
+    },
+    {
+      "epoch": 2.0255063765941483,
+      "grad_norm": 0.2312517911195755,
+      "learning_rate": 4.9883656626454724e-05,
+      "loss": 0.4782,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6132,
+      "step": 2700,
+      "tokens_per_second_per_gpu": 4275.5,
+      "total_tokens": 51921057
+    },
+    {
+      "epoch": 2.0442610652663165,
+      "grad_norm": 0.19745635986328125,
+      "learning_rate": 4.81494855608843e-05,
+      "loss": 0.4717,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6027,
+      "step": 2725,
+      "tokens_per_second_per_gpu": 4290.88,
+      "total_tokens": 52372623
+    },
+    {
+      "epoch": 2.0630157539384846,
+      "grad_norm": 0.22817276418209076,
+      "learning_rate": 4.643638525666095e-05,
+      "loss": 0.4817,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6188,
+      "step": 2750,
+      "tokens_per_second_per_gpu": 4292.31,
+      "total_tokens": 52823263
+    },
+    {
+      "epoch": 2.0817704426106527,
+      "grad_norm": 0.20878754556179047,
+      "learning_rate": 4.4745051875203134e-05,
+      "loss": 0.4774,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6119,
+      "step": 2775,
+      "tokens_per_second_per_gpu": 4287.12,
+      "total_tokens": 53272669
+    },
+    {
+      "epoch": 2.100525131282821,
+      "grad_norm": 0.18676196038722992,
+      "learning_rate": 4.307617273239226e-05,
+      "loss": 0.4824,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.62,
+      "step": 2800,
+      "tokens_per_second_per_gpu": 4304.14,
+      "total_tokens": 53724750
+    },
+    {
+      "epoch": 2.119279819954989,
+      "grad_norm": 0.20670537650585175,
+      "learning_rate": 4.1430426019264924e-05,
+      "loss": 0.4701,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6002,
+      "step": 2825,
+      "tokens_per_second_per_gpu": 4283.76,
+      "total_tokens": 54172957
+    },
+    {
+      "epoch": 2.1380345086271566,
+      "grad_norm": 0.21445906162261963,
+      "learning_rate": 3.980848052641286e-05,
+      "loss": 0.4772,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6116,
+      "step": 2850,
+      "tokens_per_second_per_gpu": 3768.93,
+      "total_tokens": 54625827
+    },
+    {
+      "epoch": 2.1567891972993247,
+      "grad_norm": 0.21021129190921783,
+      "learning_rate": 3.8210995372202896e-05,
+      "loss": 0.471,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6016,
+      "step": 2875,
+      "tokens_per_second_per_gpu": 4286.55,
+      "total_tokens": 55076031
+    },
+    {
+      "epoch": 2.175543885971493,
+      "grad_norm": 0.23069453239440918,
+      "learning_rate": 3.663861973492776e-05,
+      "loss": 0.4722,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6035,
+      "step": 2900,
+      "tokens_per_second_per_gpu": 4291.53,
+      "total_tokens": 55527864
+    },
+    {
+      "epoch": 2.194298574643661,
+      "grad_norm": 0.22328485548496246,
+      "learning_rate": 3.509199258899603e-05,
+      "loss": 0.474,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6064,
+      "step": 2925,
+      "tokens_per_second_per_gpu": 4262.17,
+      "total_tokens": 55976245
+    },
+    {
+      "epoch": 2.213053263315829,
+      "grad_norm": 0.20422938466072083,
+      "learning_rate": 3.3571742445268995e-05,
+      "loss": 0.4721,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6034,
+      "step": 2950,
+      "tokens_per_second_per_gpu": 4339.03,
+      "total_tokens": 56430293
+    },
+    {
+      "epoch": 2.231807951987997,
+      "grad_norm": 0.21462033689022064,
+      "learning_rate": 3.2078487095649236e-05,
+      "loss": 0.4798,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6158,
+      "step": 2975,
+      "tokens_per_second_per_gpu": 4274.93,
+      "total_tokens": 56879796
+    },
+    {
+      "epoch": 2.250562640660165,
+      "grad_norm": 0.21800526976585388,
+      "learning_rate": 3.061283336202545e-05,
+      "loss": 0.4733,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6053,
+      "step": 3000,
+      "tokens_per_second_per_gpu": 4290.7,
+      "total_tokens": 57329902
+    },
+    {
+      "epoch": 2.250562640660165,
+      "eval_loss": 0.49272674322128296,
+      "eval_ppl": 1.6368,
+      "eval_runtime": 139.4189,
+      "eval_samples_per_second": 5.236,
+      "eval_steps_per_second": 1.047,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 19.1,
+      "memory/max_allocated (GiB)": 19.1,
+      "step": 3000
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 3996,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.44194006269952e+18,
+  "train_batch_size": 5,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-3000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a75cafa9afe18e1aa48e766453f03add03ceb5dd78011703e6068968ae04eab2
+size 7761

checkpoint-3996/README.md

@@ -0,0 +1,208 @@
+---
+base_model: codellama/CodeLlama-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:codellama/CodeLlama-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.0

checkpoint-3996/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "codellama/CodeLlama-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-3996/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:117f1345e5c50f25128a7a46faf63217f9c93561ea40a0465fc49fe91eab762a
+size 67143296

checkpoint-3996/chat_template.jinja

@@ -0,0 +1,5 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}{% endif %}

checkpoint-3996/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58aad3dbb3cfe1b98763989bd9c75ff86952302613d5f2ee49aab4c7295f5add
+size 134433995

checkpoint-3996/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b04604ef23b357ed9d36f23ae035970360a95042e84454038a4f247858716bb3
+size 14645

checkpoint-3996/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02ee8d28df036529f2e1f752f7ac7ca36c848af464c831112a3b6510f5494afb
+size 1465

checkpoint-3996/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-3996/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ccb9c8b6b561889acea59191d66986d314e7cbd6a78abc6e49b139ca91c1e6
+size 500058

checkpoint-3996/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

checkpoint-3996/trainer_state.json

@@ -0,0 +1,2149 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9977494373593396,
+  "eval_steps": 1000,
+  "global_step": 3996,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0,
+      "eval_loss": 1.6887853145599365,
+      "eval_ppl": 5.4129,
+      "eval_runtime": 167.3526,
+      "eval_samples_per_second": 4.362,
+      "eval_steps_per_second": 0.872,
+      "memory/device_reserved (GiB)": 139.12,
+      "memory/max_active (GiB)": 18.94,
+      "memory/max_allocated (GiB)": 18.94,
+      "step": 0
+    },
+    {
+      "epoch": 0.018754688672168042,
+      "grad_norm": 1.415561556816101,
+      "learning_rate": 4.8e-05,
+      "loss": 1.6848,
+      "memory/device_reserved (GiB)": 139.11,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 5.3914,
+      "step": 25,
+      "tokens_per_second_per_gpu": 16277.76,
+      "total_tokens": 1723633
+    },
+    {
+      "epoch": 0.037509377344336084,
+      "grad_norm": 0.33179354667663574,
+      "learning_rate": 9.8e-05,
+      "loss": 0.9839,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.6749,
+      "step": 50,
+      "tokens_per_second_per_gpu": 4303.21,
+      "total_tokens": 2175386
+    },
+    {
+      "epoch": 0.056264066016504126,
+      "grad_norm": 0.17453454434871674,
+      "learning_rate": 0.000148,
+      "loss": 0.8002,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.226,
+      "step": 75,
+      "tokens_per_second_per_gpu": 3776.03,
+      "total_tokens": 2623712
+    },
+    {
+      "epoch": 0.07501875468867217,
+      "grad_norm": 0.19318008422851562,
+      "learning_rate": 0.00019800000000000002,
+      "loss": 0.7218,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 2.0581,
+      "step": 100,
+      "tokens_per_second_per_gpu": 4252.49,
+      "total_tokens": 3072519
+    },
+    {
+      "epoch": 0.09377344336084022,
+      "grad_norm": 0.18435686826705933,
+      "learning_rate": 0.00019998127418269004,
+      "loss": 0.6759,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.9658,
+      "step": 125,
+      "tokens_per_second_per_gpu": 4303.31,
+      "total_tokens": 3523983
+    },
+    {
+      "epoch": 0.11252813203300825,
+      "grad_norm": 0.19870473444461823,
+      "learning_rate": 0.00019992195096972548,
+      "loss": 0.6703,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.9548,
+      "step": 150,
+      "tokens_per_second_per_gpu": 4260.86,
+      "total_tokens": 3973452
+    },
+    {
+      "epoch": 0.1312828207051763,
+      "grad_norm": 0.20499658584594727,
+      "learning_rate": 0.0001998220219574743,
+      "loss": 0.6381,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8929,
+      "step": 175,
+      "tokens_per_second_per_gpu": 4288.64,
+      "total_tokens": 4423763
+    },
+    {
+      "epoch": 0.15003750937734434,
+      "grad_norm": 0.18934418261051178,
+      "learning_rate": 0.00019968152775460537,
+      "loss": 0.6383,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8933,
+      "step": 200,
+      "tokens_per_second_per_gpu": 4244.79,
+      "total_tokens": 4872365
+    },
+    {
+      "epoch": 0.16879219804951237,
+      "grad_norm": 0.1827855408191681,
+      "learning_rate": 0.00019950052545447352,
+      "loss": 0.6347,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8865,
+      "step": 225,
+      "tokens_per_second_per_gpu": 4252.71,
+      "total_tokens": 5319322
+    },
+    {
+      "epoch": 0.18754688672168043,
+      "grad_norm": 0.16483066976070404,
+      "learning_rate": 0.00019927908861191827,
+      "loss": 0.6392,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.895,
+      "step": 250,
+      "tokens_per_second_per_gpu": 3772.0,
+      "total_tokens": 5768644
+    },
+    {
+      "epoch": 0.20630157539384847,
+      "grad_norm": 0.17186357080936432,
+      "learning_rate": 0.00019901730721337302,
+      "loss": 0.614,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8478,
+      "step": 275,
+      "tokens_per_second_per_gpu": 4281.82,
+      "total_tokens": 6220751
+    },
+    {
+      "epoch": 0.2250562640660165,
+      "grad_norm": 0.18073013424873352,
+      "learning_rate": 0.00019871528764029667,
+      "loss": 0.6196,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8582,
+      "step": 300,
+      "tokens_per_second_per_gpu": 4234.51,
+      "total_tokens": 6668111
+    },
+    {
+      "epoch": 0.24381095273818454,
+      "grad_norm": 0.19639697670936584,
+      "learning_rate": 0.00019837315262594306,
+      "loss": 0.6181,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8554,
+      "step": 325,
+      "tokens_per_second_per_gpu": 4261.44,
+      "total_tokens": 7117439
+    },
+    {
+      "epoch": 0.2625656414103526,
+      "grad_norm": 0.1670486479997635,
+      "learning_rate": 0.00019799104120548492,
+      "loss": 0.6141,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.848,
+      "step": 350,
+      "tokens_per_second_per_gpu": 4298.97,
+      "total_tokens": 7569060
+    },
+    {
+      "epoch": 0.2813203300825206,
+      "grad_norm": 0.17752495408058167,
+      "learning_rate": 0.00019756910865951377,
+      "loss": 0.6075,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8358,
+      "step": 375,
+      "tokens_per_second_per_gpu": 4256.6,
+      "total_tokens": 8017630
+    },
+    {
+      "epoch": 0.30007501875468867,
+      "grad_norm": 0.2000180035829544,
+      "learning_rate": 0.00019710752645093747,
+      "loss": 0.6108,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8419,
+      "step": 400,
+      "tokens_per_second_per_gpu": 4245.46,
+      "total_tokens": 8464998
+    },
+    {
+      "epoch": 0.31882970742685673,
+      "grad_norm": 0.17395919561386108,
+      "learning_rate": 0.00019660648215530206,
+      "loss": 0.5966,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8159,
+      "step": 425,
+      "tokens_per_second_per_gpu": 3758.92,
+      "total_tokens": 8914723
+    },
+    {
+      "epoch": 0.33758439609902474,
+      "grad_norm": 0.18785236775875092,
+      "learning_rate": 0.00019606617938456572,
+      "loss": 0.6099,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.8402,
+      "step": 450,
+      "tokens_per_second_per_gpu": 4200.53,
+      "total_tokens": 9359638
+    },
+    {
+      "epoch": 0.3563390847711928,
+      "grad_norm": 0.17702797055244446,
+      "learning_rate": 0.0001954868377043559,
+      "loss": 0.5922,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.808,
+      "step": 475,
+      "tokens_per_second_per_gpu": 4265.36,
+      "total_tokens": 9810837
+    },
+    {
+      "epoch": 0.37509377344336087,
+      "grad_norm": 0.19927558302879333,
+      "learning_rate": 0.00019486869254474337,
+      "loss": 0.5759,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7787,
+      "step": 500,
+      "tokens_per_second_per_gpu": 4276.25,
+      "total_tokens": 10261446
+    },
+    {
+      "epoch": 0.3938484621155289,
+      "grad_norm": 0.1908370852470398,
+      "learning_rate": 0.0001942119951045692,
+      "loss": 0.584,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7932,
+      "step": 525,
+      "tokens_per_second_per_gpu": 4272.28,
+      "total_tokens": 10707841
+    },
+    {
+      "epoch": 0.41260315078769694,
+      "grad_norm": 0.2064146101474762,
+      "learning_rate": 0.00019351701224936383,
+      "loss": 0.5791,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7844,
+      "step": 550,
+      "tokens_per_second_per_gpu": 4250.37,
+      "total_tokens": 11155384
+    },
+    {
+      "epoch": 0.43135783945986494,
+      "grad_norm": 0.26748332381248474,
+      "learning_rate": 0.0001927840264028995,
+      "loss": 0.5758,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7786,
+      "step": 575,
+      "tokens_per_second_per_gpu": 4256.55,
+      "total_tokens": 11601192
+    },
+    {
+      "epoch": 0.450112528132033,
+      "grad_norm": 0.17514832317829132,
+      "learning_rate": 0.00019201333543242036,
+      "loss": 0.5791,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7844,
+      "step": 600,
+      "tokens_per_second_per_gpu": 3770.83,
+      "total_tokens": 12048477
+    },
+    {
+      "epoch": 0.46886721680420107,
+      "grad_norm": 0.22069169580936432,
+      "learning_rate": 0.00019120525252759647,
+      "loss": 0.5803,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7866,
+      "step": 625,
+      "tokens_per_second_per_gpu": 4179.31,
+      "total_tokens": 12488141
+    },
+    {
+      "epoch": 0.4876219054763691,
+      "grad_norm": 0.20555566251277924,
+      "learning_rate": 0.00019036010607325138,
+      "loss": 0.5716,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7711,
+      "step": 650,
+      "tokens_per_second_per_gpu": 4209.96,
+      "total_tokens": 12934358
+    },
+    {
+      "epoch": 0.5063765941485371,
+      "grad_norm": 0.19018156826496124,
+      "learning_rate": 0.00018947823951591478,
+      "loss": 0.5608,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7521,
+      "step": 675,
+      "tokens_per_second_per_gpu": 4226.4,
+      "total_tokens": 13378983
+    },
+    {
+      "epoch": 0.5251312828207052,
+      "grad_norm": 0.17173859477043152,
+      "learning_rate": 0.00018856001122425416,
+      "loss": 0.5667,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7624,
+      "step": 700,
+      "tokens_per_second_per_gpu": 4265.57,
+      "total_tokens": 13829519
+    },
+    {
+      "epoch": 0.5438859714928732,
+      "grad_norm": 0.17706550657749176,
+      "learning_rate": 0.0001876057943434428,
+      "loss": 0.565,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7594,
+      "step": 725,
+      "tokens_per_second_per_gpu": 4281.61,
+      "total_tokens": 14281879
+    },
+    {
+      "epoch": 0.5626406601650412,
+      "grad_norm": 0.18528586626052856,
+      "learning_rate": 0.00018661597664352284,
+      "loss": 0.5666,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7623,
+      "step": 750,
+      "tokens_per_second_per_gpu": 4229.32,
+      "total_tokens": 14725919
+    },
+    {
+      "epoch": 0.5813953488372093,
+      "grad_norm": 0.16790929436683655,
+      "learning_rate": 0.00018559096036182516,
+      "loss": 0.5633,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7565,
+      "step": 775,
+      "tokens_per_second_per_gpu": 3775.0,
+      "total_tokens": 15175146
+    },
+    {
+      "epoch": 0.6001500375093773,
+      "grad_norm": 0.17511805891990662,
+      "learning_rate": 0.00018453116203951005,
+      "loss": 0.5664,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7619,
+      "step": 800,
+      "tokens_per_second_per_gpu": 4218.07,
+      "total_tokens": 15619901
+    },
+    {
+      "epoch": 0.6189047261815454,
+      "grad_norm": 0.19853387773036957,
+      "learning_rate": 0.0001834370123522954,
+      "loss": 0.5646,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7587,
+      "step": 825,
+      "tokens_per_second_per_gpu": 4230.84,
+      "total_tokens": 16066102
+    },
+    {
+      "epoch": 0.6376594148537135,
+      "grad_norm": 0.18872258067131042,
+      "learning_rate": 0.00018230895593544056,
+      "loss": 0.552,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7367,
+      "step": 850,
+      "tokens_per_second_per_gpu": 4222.33,
+      "total_tokens": 16510696
+    },
+    {
+      "epoch": 0.6564141035258815,
+      "grad_norm": 0.9702818989753723,
+      "learning_rate": 0.0001811474512030578,
+      "loss": 0.5607,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7519,
+      "step": 875,
+      "tokens_per_second_per_gpu": 4200.39,
+      "total_tokens": 16953918
+    },
+    {
+      "epoch": 0.6751687921980495,
+      "grad_norm": 0.17479568719863892,
+      "learning_rate": 0.00017995297016182405,
+      "loss": 0.564,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7577,
+      "step": 900,
+      "tokens_per_second_per_gpu": 4210.15,
+      "total_tokens": 17396453
+    },
+    {
+      "epoch": 0.6939234808702176,
+      "grad_norm": 0.1948954463005066,
+      "learning_rate": 0.0001787259982191692,
+      "loss": 0.5511,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7352,
+      "step": 925,
+      "tokens_per_second_per_gpu": 4237.98,
+      "total_tokens": 17841287
+    },
+    {
+      "epoch": 0.7126781695423856,
+      "grad_norm": 0.19541053473949432,
+      "learning_rate": 0.00017746703398601872,
+      "loss": 0.5532,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7388,
+      "step": 950,
+      "tokens_per_second_per_gpu": 3725.33,
+      "total_tokens": 18283596
+    },
+    {
+      "epoch": 0.7314328582145536,
+      "grad_norm": 0.1818365603685379,
+      "learning_rate": 0.0001761765890741701,
+      "loss": 0.5521,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7369,
+      "step": 975,
+      "tokens_per_second_per_gpu": 4211.63,
+      "total_tokens": 18726722
+    },
+    {
+      "epoch": 0.7501875468867217,
+      "grad_norm": 0.1838025599718094,
+      "learning_rate": 0.00017485518788838705,
+      "loss": 0.5511,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7352,
+      "step": 1000,
+      "tokens_per_second_per_gpu": 3962.4,
+      "total_tokens": 19167258
+    },
+    {
+      "epoch": 0.7501875468867217,
+      "eval_loss": 0.540988564491272,
+      "eval_ppl": 1.7177,
+      "eval_runtime": 138.0264,
+      "eval_samples_per_second": 5.289,
+      "eval_steps_per_second": 1.058,
+      "memory/device_reserved (GiB)": 139.02,
+      "memory/max_active (GiB)": 19.1,
+      "memory/max_allocated (GiB)": 19.1,
+      "step": 1000
+    },
+    {
+      "epoch": 0.7689422355588897,
+      "grad_norm": 0.2199818342924118,
+      "learning_rate": 0.00017350336741329413,
+      "loss": 0.549,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7315,
+      "step": 1025,
+      "tokens_per_second_per_gpu": 4129.73,
+      "total_tokens": 20870820
+    },
+    {
+      "epoch": 0.7876969242310577,
+      "grad_norm": 0.19783177971839905,
+      "learning_rate": 0.0001721216769951596,
+      "loss": 0.5615,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7533,
+      "step": 1050,
+      "tokens_per_second_per_gpu": 4243.63,
+      "total_tokens": 21317982
+    },
+    {
+      "epoch": 0.8064516129032258,
+      "grad_norm": 0.1678430140018463,
+      "learning_rate": 0.00017071067811865476,
+      "loss": 0.5557,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7432,
+      "step": 1075,
+      "tokens_per_second_per_gpu": 4092.04,
+      "total_tokens": 21754087
+    },
+    {
+      "epoch": 0.8252063015753939,
+      "grad_norm": 0.16523879766464233,
+      "learning_rate": 0.00016927094417868048,
+      "loss": 0.556,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7437,
+      "step": 1100,
+      "tokens_per_second_per_gpu": 4187.02,
+      "total_tokens": 22198779
+    },
+    {
+      "epoch": 0.8439609902475619,
+      "grad_norm": 0.18177717924118042,
+      "learning_rate": 0.00016780306024735382,
+      "loss": 0.5468,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7277,
+      "step": 1125,
+      "tokens_per_second_per_gpu": 4198.97,
+      "total_tokens": 22639769
+    },
+    {
+      "epoch": 0.8627156789197299,
+      "grad_norm": 0.17299720644950867,
+      "learning_rate": 0.0001663076228362492,
+      "loss": 0.554,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7402,
+      "step": 1150,
+      "tokens_per_second_per_gpu": 3762.13,
+      "total_tokens": 23086742
+    },
+    {
+      "epoch": 0.881470367591898,
+      "grad_norm": 0.19112971425056458,
+      "learning_rate": 0.00016478523965399085,
+      "loss": 0.5434,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7219,
+      "step": 1175,
+      "tokens_per_second_per_gpu": 4205.37,
+      "total_tokens": 23528106
+    },
+    {
+      "epoch": 0.900225056264066,
+      "grad_norm": 0.17930163443088531,
+      "learning_rate": 0.00016323652935929536,
+      "loss": 0.5362,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7095,
+      "step": 1200,
+      "tokens_per_second_per_gpu": 4228.83,
+      "total_tokens": 23974427
+    },
+    {
+      "epoch": 0.918979744936234,
+      "grad_norm": 0.18718039989471436,
+      "learning_rate": 0.00016166212130956382,
+      "loss": 0.5533,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.739,
+      "step": 1225,
+      "tokens_per_second_per_gpu": 4211.64,
+      "total_tokens": 24415919
+    },
+    {
+      "epoch": 0.9377344336084021,
+      "grad_norm": 0.17105573415756226,
+      "learning_rate": 0.0001600626553051268,
+      "loss": 0.5492,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7319,
+      "step": 1250,
+      "tokens_per_second_per_gpu": 4183.86,
+      "total_tokens": 24854345
+    },
+    {
+      "epoch": 0.9564891222805701,
+      "grad_norm": 0.1733955442905426,
+      "learning_rate": 0.0001584387813292454,
+      "loss": 0.5348,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7071,
+      "step": 1275,
+      "tokens_per_second_per_gpu": 4172.93,
+      "total_tokens": 25292647
+    },
+    {
+      "epoch": 0.9752438109527382,
+      "grad_norm": 0.1858205944299698,
+      "learning_rate": 0.00015679115928397401,
+      "loss": 0.5527,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7379,
+      "step": 1300,
+      "tokens_per_second_per_gpu": 4226.34,
+      "total_tokens": 25733591
+    },
+    {
+      "epoch": 0.9939984996249063,
+      "grad_norm": 0.1944192498922348,
+      "learning_rate": 0.00015512045872199276,
+      "loss": 0.5311,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.7008,
+      "step": 1325,
+      "tokens_per_second_per_gpu": 3655.12,
+      "total_tokens": 26164528
+    },
+    {
+      "epoch": 1.0127531882970742,
+      "grad_norm": 0.18358173966407776,
+      "learning_rate": 0.00015342735857451777,
+      "loss": 0.5145,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6728,
+      "step": 1350,
+      "tokens_per_second_per_gpu": 4227.25,
+      "total_tokens": 26610460
+    },
+    {
+      "epoch": 1.0315078769692423,
+      "grad_norm": 0.1853465735912323,
+      "learning_rate": 0.00015171254687540038,
+      "loss": 0.5081,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6621,
+      "step": 1375,
+      "tokens_per_second_per_gpu": 4318.88,
+      "total_tokens": 27064008
+    },
+    {
+      "epoch": 1.0502625656414104,
+      "grad_norm": 0.18925060331821442,
+      "learning_rate": 0.0001499767204815273,
+      "loss": 0.5185,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6795,
+      "step": 1400,
+      "tokens_per_second_per_gpu": 4324.01,
+      "total_tokens": 27516590
+    },
+    {
+      "epoch": 1.0690172543135783,
+      "grad_norm": 0.20961470901966095,
+      "learning_rate": 0.00014822058478963532,
+      "loss": 0.5234,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6878,
+      "step": 1425,
+      "tokens_per_second_per_gpu": 4319.64,
+      "total_tokens": 27970075
+    },
+    {
+      "epoch": 1.0877719429857464,
+      "grad_norm": 0.1982697695493698,
+      "learning_rate": 0.0001464448534496555,
+      "loss": 0.5169,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6768,
+      "step": 1450,
+      "tokens_per_second_per_gpu": 4267.88,
+      "total_tokens": 28419716
+    },
+    {
+      "epoch": 1.1065266316579145,
+      "grad_norm": 0.1925143301486969,
+      "learning_rate": 0.00014465024807470376,
+      "loss": 0.5197,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6815,
+      "step": 1475,
+      "tokens_per_second_per_gpu": 4264.53,
+      "total_tokens": 28866312
+    },
+    {
+      "epoch": 1.1252813203300824,
+      "grad_norm": 0.18788637220859528,
+      "learning_rate": 0.0001428374979478349,
+      "loss": 0.5204,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6827,
+      "step": 1500,
+      "tokens_per_second_per_gpu": 3779.33,
+      "total_tokens": 29315968
+    },
+    {
+      "epoch": 1.1440360090022506,
+      "grad_norm": 0.18954145908355713,
+      "learning_rate": 0.00014100733972568038,
+      "loss": 0.5164,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.676,
+      "step": 1525,
+      "tokens_per_second_per_gpu": 4282.57,
+      "total_tokens": 29766723
+    },
+    {
+      "epoch": 1.1627906976744187,
+      "grad_norm": 0.19003146886825562,
+      "learning_rate": 0.00013916051713908924,
+      "loss": 0.5095,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6645,
+      "step": 1550,
+      "tokens_per_second_per_gpu": 4290.76,
+      "total_tokens": 30218573
+    },
+    {
+      "epoch": 1.1815453863465866,
+      "grad_norm": 0.18279583752155304,
+      "learning_rate": 0.00013729778069089437,
+      "loss": 0.522,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6854,
+      "step": 1575,
+      "tokens_per_second_per_gpu": 4300.13,
+      "total_tokens": 30669810
+    },
+    {
+      "epoch": 1.2003000750187547,
+      "grad_norm": 0.18783092498779297,
+      "learning_rate": 0.00013541988735092672,
+      "loss": 0.5003,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6492,
+      "step": 1600,
+      "tokens_per_second_per_gpu": 4271.27,
+      "total_tokens": 31117586
+    },
+    {
+      "epoch": 1.2190547636909228,
+      "grad_norm": 0.199558824300766,
+      "learning_rate": 0.00013352760024840175,
+      "loss": 0.5115,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6678,
+      "step": 1625,
+      "tokens_per_second_per_gpu": 4248.14,
+      "total_tokens": 31562224
+    },
+    {
+      "epoch": 1.2378094523630907,
+      "grad_norm": 0.19465653598308563,
+      "learning_rate": 0.00013162168836180246,
+      "loss": 0.4967,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6433,
+      "step": 1650,
+      "tokens_per_second_per_gpu": 4286.24,
+      "total_tokens": 32011071
+    },
+    {
+      "epoch": 1.2565641410352588,
+      "grad_norm": 0.2054641842842102,
+      "learning_rate": 0.00012970292620638574,
+      "loss": 0.5172,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6773,
+      "step": 1675,
+      "tokens_per_second_per_gpu": 3733.1,
+      "total_tokens": 32452490
+    },
+    {
+      "epoch": 1.275318829707427,
+      "grad_norm": 0.19450411200523376,
+      "learning_rate": 0.00012777209351943862,
+      "loss": 0.5149,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6735,
+      "step": 1700,
+      "tokens_per_second_per_gpu": 4251.33,
+      "total_tokens": 32899103
+    },
+    {
+      "epoch": 1.2940735183795948,
+      "grad_norm": 0.19844166934490204,
+      "learning_rate": 0.0001258299749434123,
+      "loss": 0.5205,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6829,
+      "step": 1725,
+      "tokens_per_second_per_gpu": 4240.57,
+      "total_tokens": 33344569
+    },
+    {
+      "epoch": 1.312828207051763,
+      "grad_norm": 0.19240470230579376,
+      "learning_rate": 0.00012387735970706312,
+      "loss": 0.5033,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6542,
+      "step": 1750,
+      "tokens_per_second_per_gpu": 4267.65,
+      "total_tokens": 33790426
+    },
+    {
+      "epoch": 1.331582895723931,
+      "grad_norm": 0.18220192193984985,
+      "learning_rate": 0.00012191504130472937,
+      "loss": 0.5103,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6658,
+      "step": 1775,
+      "tokens_per_second_per_gpu": 4237.08,
+      "total_tokens": 34233908
+    },
+    {
+      "epoch": 1.350337584396099,
+      "grad_norm": 0.20157551765441895,
+      "learning_rate": 0.00011994381717387514,
+      "loss": 0.5192,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6807,
+      "step": 1800,
+      "tokens_per_second_per_gpu": 4244.09,
+      "total_tokens": 34678691
+    },
+    {
+      "epoch": 1.369092273068267,
+      "grad_norm": 0.17189238965511322,
+      "learning_rate": 0.00011796448837103129,
+      "loss": 0.5011,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6505,
+      "step": 1825,
+      "tokens_per_second_per_gpu": 4277.26,
+      "total_tokens": 35125624
+    },
+    {
+      "epoch": 1.387846961740435,
+      "grad_norm": 0.19443106651306152,
+      "learning_rate": 0.00011597785924626616,
+      "loss": 0.4994,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6477,
+      "step": 1850,
+      "tokens_per_second_per_gpu": 3766.52,
+      "total_tokens": 35568850
+    },
+    {
+      "epoch": 1.406601650412603,
+      "grad_norm": 0.1810811311006546,
+      "learning_rate": 0.00011398473711631764,
+      "loss": 0.5083,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6625,
+      "step": 1875,
+      "tokens_per_second_per_gpu": 4204.76,
+      "total_tokens": 36009980
+    },
+    {
+      "epoch": 1.4253563390847712,
+      "grad_norm": 0.19805970788002014,
+      "learning_rate": 0.00011198593193651958,
+      "loss": 0.5141,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6721,
+      "step": 1900,
+      "tokens_per_second_per_gpu": 4270.21,
+      "total_tokens": 36457032
+    },
+    {
+      "epoch": 1.4441110277569393,
+      "grad_norm": 0.1936168372631073,
+      "learning_rate": 0.00010998225597165628,
+      "loss": 0.5045,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6562,
+      "step": 1925,
+      "tokens_per_second_per_gpu": 4275.24,
+      "total_tokens": 36905590
+    },
+    {
+      "epoch": 1.4628657164291072,
+      "grad_norm": 0.19065748155117035,
+      "learning_rate": 0.00010797452346587798,
+      "loss": 0.5025,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6528,
+      "step": 1950,
+      "tokens_per_second_per_gpu": 4285.81,
+      "total_tokens": 37354436
+    },
+    {
+      "epoch": 1.4816204051012754,
+      "grad_norm": 0.18647657334804535,
+      "learning_rate": 0.0001059635503118125,
+      "loss": 0.5102,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6656,
+      "step": 1975,
+      "tokens_per_second_per_gpu": 4259.76,
+      "total_tokens": 37801500
+    },
+    {
+      "epoch": 1.5003750937734432,
+      "grad_norm": 0.21211788058280945,
+      "learning_rate": 0.00010395015371900663,
+      "loss": 0.5052,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6573,
+      "step": 2000,
+      "tokens_per_second_per_gpu": 4250.7,
+      "total_tokens": 38244936
+    },
+    {
+      "epoch": 1.5003750937734432,
+      "eval_loss": 0.5063687562942505,
+      "eval_ppl": 1.6593,
+      "eval_runtime": 141.112,
+      "eval_samples_per_second": 5.173,
+      "eval_steps_per_second": 1.035,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 19.1,
+      "memory/max_allocated (GiB)": 19.1,
+      "step": 2000
+    },
+    {
+      "epoch": 1.5191297824456114,
+      "grad_norm": 0.20089760422706604,
+      "learning_rate": 0.00010193515188183245,
+      "loss": 0.4892,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.631,
+      "step": 2025,
+      "tokens_per_second_per_gpu": 4246.58,
+      "total_tokens": 39959888
+    },
+    {
+      "epoch": 1.5378844711177795,
+      "grad_norm": 0.19840118288993835,
+      "learning_rate": 9.991936364699348e-05,
+      "loss": 0.503,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6537,
+      "step": 2050,
+      "tokens_per_second_per_gpu": 4320.38,
+      "total_tokens": 40411902
+    },
+    {
+      "epoch": 1.5566391597899476,
+      "grad_norm": 0.20045842230319977,
+      "learning_rate": 9.790360818076577e-05,
+      "loss": 0.5127,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6698,
+      "step": 2075,
+      "tokens_per_second_per_gpu": 4245.02,
+      "total_tokens": 40855384
+    },
+    {
+      "epoch": 1.5753938484621155,
+      "grad_norm": 0.19669026136398315,
+      "learning_rate": 9.588870463610893e-05,
+      "loss": 0.4994,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6477,
+      "step": 2100,
+      "tokens_per_second_per_gpu": 4174.18,
+      "total_tokens": 41293525
+    },
+    {
+      "epoch": 1.5941485371342836,
+      "grad_norm": 0.19754259288311005,
+      "learning_rate": 9.387547181978291e-05,
+      "loss": 0.5009,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6502,
+      "step": 2125,
+      "tokens_per_second_per_gpu": 4200.06,
+      "total_tokens": 41737747
+    },
+    {
+      "epoch": 1.6129032258064515,
+      "grad_norm": 0.19482502341270447,
+      "learning_rate": 9.186472785960507e-05,
+      "loss": 0.5002,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6491,
+      "step": 2150,
+      "tokens_per_second_per_gpu": 3696.76,
+      "total_tokens": 42176082
+    },
+    {
+      "epoch": 1.6316579144786196,
+      "grad_norm": 0.21606561541557312,
+      "learning_rate": 8.985728987198352e-05,
+      "loss": 0.4959,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.642,
+      "step": 2175,
+      "tokens_per_second_per_gpu": 4192.5,
+      "total_tokens": 42616372
+    },
+    {
+      "epoch": 1.6504126031507877,
+      "grad_norm": 0.1979638934135437,
+      "learning_rate": 8.785397362986114e-05,
+      "loss": 0.5031,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6538,
+      "step": 2200,
+      "tokens_per_second_per_gpu": 4211.67,
+      "total_tokens": 43058315
+    },
+    {
+      "epoch": 1.6691672918229559,
+      "grad_norm": 0.20717743039131165,
+      "learning_rate": 8.58555932312059e-05,
+      "loss": 0.4986,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6464,
+      "step": 2225,
+      "tokens_per_second_per_gpu": 4242.04,
+      "total_tokens": 43501960
+    },
+    {
+      "epoch": 1.6879219804951238,
+      "grad_norm": 0.18736609816551208,
+      "learning_rate": 8.38629607681815e-05,
+      "loss": 0.4898,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.632,
+      "step": 2250,
+      "tokens_per_second_per_gpu": 4235.21,
+      "total_tokens": 43947235
+    },
+    {
+      "epoch": 1.7066766691672917,
+      "grad_norm": 0.2056591659784317,
+      "learning_rate": 8.187688599713333e-05,
+      "loss": 0.4925,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6364,
+      "step": 2275,
+      "tokens_per_second_per_gpu": 4256.41,
+      "total_tokens": 44393451
+    },
+    {
+      "epoch": 1.7254313578394598,
+      "grad_norm": 0.19774597883224487,
+      "learning_rate": 7.989817600952376e-05,
+      "loss": 0.4952,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6408,
+      "step": 2300,
+      "tokens_per_second_per_gpu": 4224.5,
+      "total_tokens": 44836590
+    },
+    {
+      "epoch": 1.744186046511628,
+      "grad_norm": 0.19662383198738098,
+      "learning_rate": 7.792763490394984e-05,
+      "loss": 0.4977,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6449,
+      "step": 2325,
+      "tokens_per_second_per_gpu": 3741.52,
+      "total_tokens": 45279799
+    },
+    {
+      "epoch": 1.762940735183796,
+      "grad_norm": 0.19400179386138916,
+      "learning_rate": 7.596606345937812e-05,
+      "loss": 0.4965,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.643,
+      "step": 2350,
+      "tokens_per_second_per_gpu": 4248.51,
+      "total_tokens": 45725602
+    },
+    {
+      "epoch": 1.7816954238559641,
+      "grad_norm": 0.20261766016483307,
+      "learning_rate": 7.401425880972742e-05,
+      "loss": 0.5014,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.651,
+      "step": 2375,
+      "tokens_per_second_per_gpu": 4216.2,
+      "total_tokens": 46167730
+    },
+    {
+      "epoch": 1.800450112528132,
+      "grad_norm": 0.20447255671024323,
+      "learning_rate": 7.207301411993387e-05,
+      "loss": 0.4901,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6325,
+      "step": 2400,
+      "tokens_per_second_per_gpu": 3727.37,
+      "total_tokens": 46611126
+    },
+    {
+      "epoch": 1.8192048012003,
+      "grad_norm": 0.19921696186065674,
+      "learning_rate": 7.014311826362804e-05,
+      "loss": 0.4925,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6364,
+      "step": 2425,
+      "tokens_per_second_per_gpu": 4202.19,
+      "total_tokens": 47050763
+    },
+    {
+      "epoch": 1.837959489872468,
+      "grad_norm": 0.20095540583133698,
+      "learning_rate": 6.822535550255652e-05,
+      "loss": 0.494,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6389,
+      "step": 2450,
+      "tokens_per_second_per_gpu": 4230.16,
+      "total_tokens": 47496926
+    },
+    {
+      "epoch": 1.8567141785446362,
+      "grad_norm": 0.20210741460323334,
+      "learning_rate": 6.632050516787719e-05,
+      "loss": 0.5036,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6547,
+      "step": 2475,
+      "tokens_per_second_per_gpu": 4256.1,
+      "total_tokens": 47941250
+    },
+    {
+      "epoch": 1.8754688672168043,
+      "grad_norm": 0.21025419235229492,
+      "learning_rate": 6.442934134345871e-05,
+      "loss": 0.5019,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6519,
+      "step": 2500,
+      "tokens_per_second_per_gpu": 3728.09,
+      "total_tokens": 48383306
+    },
+    {
+      "epoch": 1.8942235558889724,
+      "grad_norm": 0.20130059123039246,
+      "learning_rate": 6.255263255131172e-05,
+      "loss": 0.5022,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6524,
+      "step": 2525,
+      "tokens_per_second_per_gpu": 4178.95,
+      "total_tokens": 48821862
+    },
+    {
+      "epoch": 1.9129782445611403,
+      "grad_norm": 0.19601669907569885,
+      "learning_rate": 6.0691141439280785e-05,
+      "loss": 0.4876,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6284,
+      "step": 2550,
+      "tokens_per_second_per_gpu": 3998.52,
+      "total_tokens": 49262344
+    },
+    {
+      "epoch": 1.9317329332333082,
+      "grad_norm": 0.20538586378097534,
+      "learning_rate": 5.884562447112331e-05,
+      "loss": 0.4796,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6154,
+      "step": 2575,
+      "tokens_per_second_per_gpu": 4192.8,
+      "total_tokens": 49702209
+    },
+    {
+      "epoch": 1.9504876219054763,
+      "grad_norm": 0.19957959651947021,
+      "learning_rate": 5.701683161910115e-05,
+      "loss": 0.5017,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6515,
+      "step": 2600,
+      "tokens_per_second_per_gpu": 4244.94,
+      "total_tokens": 50147673
+    },
+    {
+      "epoch": 1.9692423105776444,
+      "grad_norm": 0.20284536480903625,
+      "learning_rate": 5.520550605921091e-05,
+      "loss": 0.5024,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6527,
+      "step": 2625,
+      "tokens_per_second_per_gpu": 4205.45,
+      "total_tokens": 50589478
+    },
+    {
+      "epoch": 1.9879969992498125,
+      "grad_norm": 0.2044789344072342,
+      "learning_rate": 5.34123838691753e-05,
+      "loss": 0.4967,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6433,
+      "step": 2650,
+      "tokens_per_second_per_gpu": 4204.9,
+      "total_tokens": 51027800
+    },
+    {
+      "epoch": 2.0067516879219807,
+      "grad_norm": 0.2125943899154663,
+      "learning_rate": 5.163819372931979e-05,
+      "loss": 0.4862,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6261,
+      "step": 2675,
+      "tokens_per_second_per_gpu": 3745.54,
+      "total_tokens": 51469941
+    },
+    {
+      "epoch": 2.0255063765941483,
+      "grad_norm": 0.2312517911195755,
+      "learning_rate": 4.9883656626454724e-05,
+      "loss": 0.4782,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6132,
+      "step": 2700,
+      "tokens_per_second_per_gpu": 4275.5,
+      "total_tokens": 51921057
+    },
+    {
+      "epoch": 2.0442610652663165,
+      "grad_norm": 0.19745635986328125,
+      "learning_rate": 4.81494855608843e-05,
+      "loss": 0.4717,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6027,
+      "step": 2725,
+      "tokens_per_second_per_gpu": 4290.88,
+      "total_tokens": 52372623
+    },
+    {
+      "epoch": 2.0630157539384846,
+      "grad_norm": 0.22817276418209076,
+      "learning_rate": 4.643638525666095e-05,
+      "loss": 0.4817,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6188,
+      "step": 2750,
+      "tokens_per_second_per_gpu": 4292.31,
+      "total_tokens": 52823263
+    },
+    {
+      "epoch": 2.0817704426106527,
+      "grad_norm": 0.20878754556179047,
+      "learning_rate": 4.4745051875203134e-05,
+      "loss": 0.4774,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6119,
+      "step": 2775,
+      "tokens_per_second_per_gpu": 4287.12,
+      "total_tokens": 53272669
+    },
+    {
+      "epoch": 2.100525131282821,
+      "grad_norm": 0.18676196038722992,
+      "learning_rate": 4.307617273239226e-05,
+      "loss": 0.4824,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.62,
+      "step": 2800,
+      "tokens_per_second_per_gpu": 4304.14,
+      "total_tokens": 53724750
+    },
+    {
+      "epoch": 2.119279819954989,
+      "grad_norm": 0.20670537650585175,
+      "learning_rate": 4.1430426019264924e-05,
+      "loss": 0.4701,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6002,
+      "step": 2825,
+      "tokens_per_second_per_gpu": 4283.76,
+      "total_tokens": 54172957
+    },
+    {
+      "epoch": 2.1380345086271566,
+      "grad_norm": 0.21445906162261963,
+      "learning_rate": 3.980848052641286e-05,
+      "loss": 0.4772,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6116,
+      "step": 2850,
+      "tokens_per_second_per_gpu": 3768.93,
+      "total_tokens": 54625827
+    },
+    {
+      "epoch": 2.1567891972993247,
+      "grad_norm": 0.21021129190921783,
+      "learning_rate": 3.8210995372202896e-05,
+      "loss": 0.471,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6016,
+      "step": 2875,
+      "tokens_per_second_per_gpu": 4286.55,
+      "total_tokens": 55076031
+    },
+    {
+      "epoch": 2.175543885971493,
+      "grad_norm": 0.23069453239440918,
+      "learning_rate": 3.663861973492776e-05,
+      "loss": 0.4722,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6035,
+      "step": 2900,
+      "tokens_per_second_per_gpu": 4291.53,
+      "total_tokens": 55527864
+    },
+    {
+      "epoch": 2.194298574643661,
+      "grad_norm": 0.22328485548496246,
+      "learning_rate": 3.509199258899603e-05,
+      "loss": 0.474,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6064,
+      "step": 2925,
+      "tokens_per_second_per_gpu": 4262.17,
+      "total_tokens": 55976245
+    },
+    {
+      "epoch": 2.213053263315829,
+      "grad_norm": 0.20422938466072083,
+      "learning_rate": 3.3571742445268995e-05,
+      "loss": 0.4721,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6034,
+      "step": 2950,
+      "tokens_per_second_per_gpu": 4339.03,
+      "total_tokens": 56430293
+    },
+    {
+      "epoch": 2.231807951987997,
+      "grad_norm": 0.21462033689022064,
+      "learning_rate": 3.2078487095649236e-05,
+      "loss": 0.4798,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6158,
+      "step": 2975,
+      "tokens_per_second_per_gpu": 4274.93,
+      "total_tokens": 56879796
+    },
+    {
+      "epoch": 2.250562640660165,
+      "grad_norm": 0.21800526976585388,
+      "learning_rate": 3.061283336202545e-05,
+      "loss": 0.4733,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6053,
+      "step": 3000,
+      "tokens_per_second_per_gpu": 4290.7,
+      "total_tokens": 57329902
+    },
+    {
+      "epoch": 2.250562640660165,
+      "eval_loss": 0.49272674322128296,
+      "eval_ppl": 1.6368,
+      "eval_runtime": 139.4189,
+      "eval_samples_per_second": 5.236,
+      "eval_steps_per_second": 1.047,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 19.1,
+      "memory/max_allocated (GiB)": 19.1,
+      "step": 3000
+    },
+    {
+      "epoch": 2.269317329332333,
+      "grad_norm": 0.23463094234466553,
+      "learning_rate": 2.9175376849675073e-05,
+      "loss": 0.4705,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6008,
+      "step": 3025,
+      "tokens_per_second_per_gpu": 4276.0,
+      "total_tokens": 59047769
+    },
+    {
+      "epoch": 2.288072018004501,
+      "grad_norm": 0.2144247442483902,
+      "learning_rate": 2.7766701705225194e-05,
+      "loss": 0.4761,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6098,
+      "step": 3050,
+      "tokens_per_second_per_gpu": 4257.17,
+      "total_tokens": 59495040
+    },
+    {
+      "epoch": 2.3068267066766692,
+      "grad_norm": 0.21562626957893372,
+      "learning_rate": 2.6387380379269623e-05,
+      "loss": 0.4576,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5803,
+      "step": 3075,
+      "tokens_per_second_per_gpu": 4276.33,
+      "total_tokens": 59941839
+    },
+    {
+      "epoch": 2.3255813953488373,
+      "grad_norm": 0.2173856496810913,
+      "learning_rate": 2.5037973393739433e-05,
+      "loss": 0.4578,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5806,
+      "step": 3100,
+      "tokens_per_second_per_gpu": 4284.31,
+      "total_tokens": 60392267
+    },
+    {
+      "epoch": 2.3443360840210055,
+      "grad_norm": 0.21864096820354462,
+      "learning_rate": 2.3719029114120716e-05,
+      "loss": 0.4652,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5923,
+      "step": 3125,
+      "tokens_per_second_per_gpu": 4250.05,
+      "total_tokens": 60836393
+    },
+    {
+      "epoch": 2.363090772693173,
+      "grad_norm": 0.22768662869930267,
+      "learning_rate": 2.2431083526612373e-05,
+      "loss": 0.4721,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6034,
+      "step": 3150,
+      "tokens_per_second_per_gpu": 3777.63,
+      "total_tokens": 61282878
+    },
+    {
+      "epoch": 2.3818454613653413,
+      "grad_norm": 0.20161285996437073,
+      "learning_rate": 2.1174660020314696e-05,
+      "loss": 0.486,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6258,
+      "step": 3175,
+      "tokens_per_second_per_gpu": 4279.7,
+      "total_tokens": 61731262
+    },
+    {
+      "epoch": 2.4006001500375094,
+      "grad_norm": 0.2132490575313568,
+      "learning_rate": 1.9950269174537007e-05,
+      "loss": 0.4822,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6196,
+      "step": 3200,
+      "tokens_per_second_per_gpu": 4245.52,
+      "total_tokens": 62179118
+    },
+    {
+      "epoch": 2.4193548387096775,
+      "grad_norm": 0.24689531326293945,
+      "learning_rate": 1.8758408551311047e-05,
+      "loss": 0.49,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6323,
+      "step": 3225,
+      "tokens_per_second_per_gpu": 4237.8,
+      "total_tokens": 62624159
+    },
+    {
+      "epoch": 2.4381095273818456,
+      "grad_norm": 0.2103738784790039,
+      "learning_rate": 1.7599562493193867e-05,
+      "loss": 0.4654,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5927,
+      "step": 3250,
+      "tokens_per_second_per_gpu": 4234.16,
+      "total_tokens": 63069936
+    },
+    {
+      "epoch": 2.4568642160540133,
+      "grad_norm": 0.2107544094324112,
+      "learning_rate": 1.6474201926443267e-05,
+      "loss": 0.4575,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5801,
+      "step": 3275,
+      "tokens_per_second_per_gpu": 4198.5,
+      "total_tokens": 63512282
+    },
+    {
+      "epoch": 2.4756189047261814,
+      "grad_norm": 0.211527019739151,
+      "learning_rate": 1.5382784169644925e-05,
+      "loss": 0.4654,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5927,
+      "step": 3300,
+      "tokens_per_second_per_gpu": 4266.77,
+      "total_tokens": 63959153
+    },
+    {
+      "epoch": 2.4943735933983495,
+      "grad_norm": 0.22054381668567657,
+      "learning_rate": 1.4325752747869626e-05,
+      "loss": 0.4601,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5842,
+      "step": 3325,
+      "tokens_per_second_per_gpu": 3810.76,
+      "total_tokens": 64408084
+    },
+    {
+      "epoch": 2.5131282820705176,
+      "grad_norm": 0.21859121322631836,
+      "learning_rate": 1.3303537212435469e-05,
+      "loss": 0.4594,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5831,
+      "step": 3350,
+      "tokens_per_second_per_gpu": 4141.07,
+      "total_tokens": 64850022
+    },
+    {
+      "epoch": 2.5318829707426858,
+      "grad_norm": 0.22012574970722198,
+      "learning_rate": 1.231655296634906e-05,
+      "loss": 0.4635,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5896,
+      "step": 3375,
+      "tokens_per_second_per_gpu": 4210.83,
+      "total_tokens": 65292271
+    },
+    {
+      "epoch": 2.550637659414854,
+      "grad_norm": 0.21981129050254822,
+      "learning_rate": 1.1365201095496048e-05,
+      "loss": 0.4809,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6175,
+      "step": 3400,
+      "tokens_per_second_per_gpu": 4233.11,
+      "total_tokens": 65735025
+    },
+    {
+      "epoch": 2.569392348087022,
+      "grad_norm": 0.22363677620887756,
+      "learning_rate": 1.0449868205649649e-05,
+      "loss": 0.4605,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5849,
+      "step": 3425,
+      "tokens_per_second_per_gpu": 4245.88,
+      "total_tokens": 66180426
+    },
+    {
+      "epoch": 2.5881470367591897,
+      "grad_norm": 0.21145139634609222,
+      "learning_rate": 9.570926265363789e-06,
+      "loss": 0.4661,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5938,
+      "step": 3450,
+      "tokens_per_second_per_gpu": 4280.85,
+      "total_tokens": 66629602
+    },
+    {
+      "epoch": 2.606901725431358,
+      "grad_norm": 0.2377360314130783,
+      "learning_rate": 8.728732454814203e-06,
+      "loss": 0.46,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5841,
+      "step": 3475,
+      "tokens_per_second_per_gpu": 4258.29,
+      "total_tokens": 67075180
+    },
+    {
+      "epoch": 2.625656414103526,
+      "grad_norm": 0.22640903294086456,
+      "learning_rate": 7.923629020649448e-06,
+      "loss": 0.4734,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6054,
+      "step": 3500,
+      "tokens_per_second_per_gpu": 3732.79,
+      "total_tokens": 67519655
+    },
+    {
+      "epoch": 2.644411102775694,
+      "grad_norm": 0.2617396414279938,
+      "learning_rate": 7.155943136910193e-06,
+      "loss": 0.4701,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6002,
+      "step": 3525,
+      "tokens_per_second_per_gpu": 4208.25,
+      "total_tokens": 67960790
+    },
+    {
+      "epoch": 2.663165791447862,
+      "grad_norm": 0.2119966447353363,
+      "learning_rate": 6.425986772073922e-06,
+      "loss": 0.4754,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6087,
+      "step": 3550,
+      "tokens_per_second_per_gpu": 4238.47,
+      "total_tokens": 68405914
+    },
+    {
+      "epoch": 2.68192048012003,
+      "grad_norm": 0.21404898166656494,
+      "learning_rate": 5.734056562278634e-06,
+      "loss": 0.4536,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.574,
+      "step": 3575,
+      "tokens_per_second_per_gpu": 4293.79,
+      "total_tokens": 68854437
+    },
+    {
+      "epoch": 2.700675168792198,
+      "grad_norm": 0.207435742020607,
+      "learning_rate": 5.080433690777353e-06,
+      "loss": 0.4726,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6042,
+      "step": 3600,
+      "tokens_per_second_per_gpu": 4137.98,
+      "total_tokens": 69296241
+    },
+    {
+      "epoch": 2.719429857464366,
+      "grad_norm": 0.23212255537509918,
+      "learning_rate": 4.465383773672127e-06,
+      "loss": 0.4626,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5882,
+      "step": 3625,
+      "tokens_per_second_per_gpu": 4195.93,
+      "total_tokens": 69736200
+    },
+    {
+      "epoch": 2.738184546136534,
+      "grad_norm": 0.24078768491744995,
+      "learning_rate": 3.889156751974343e-06,
+      "loss": 0.4652,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5923,
+      "step": 3650,
+      "tokens_per_second_per_gpu": 4260.72,
+      "total_tokens": 70181446
+    },
+    {
+      "epoch": 2.7569392348087023,
+      "grad_norm": 0.20761160552501678,
+      "learning_rate": 3.3519867900349113e-06,
+      "loss": 0.4603,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5845,
+      "step": 3675,
+      "tokens_per_second_per_gpu": 3783.05,
+      "total_tokens": 70627535
+    },
+    {
+      "epoch": 2.77569392348087,
+      "grad_norm": 0.2079222947359085,
+      "learning_rate": 2.8540921803855926e-06,
+      "loss": 0.459,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5825,
+      "step": 3700,
+      "tokens_per_second_per_gpu": 4196.92,
+      "total_tokens": 71068359
+    },
+    {
+      "epoch": 2.7944486121530385,
+      "grad_norm": 0.23349842429161072,
+      "learning_rate": 2.395675255030383e-06,
+      "loss": 0.4692,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5987,
+      "step": 3725,
+      "tokens_per_second_per_gpu": 4216.77,
+      "total_tokens": 71509553
+    },
+    {
+      "epoch": 2.813203300825206,
+      "grad_norm": 0.2154284566640854,
+      "learning_rate": 1.9769223032228724e-06,
+      "loss": 0.4626,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5882,
+      "step": 3750,
+      "tokens_per_second_per_gpu": 4278.81,
+      "total_tokens": 71956413
+    },
+    {
+      "epoch": 2.8319579894973743,
+      "grad_norm": 0.2559005916118622,
+      "learning_rate": 1.5980034957628231e-06,
+      "loss": 0.4757,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6091,
+      "step": 3775,
+      "tokens_per_second_per_gpu": 4161.74,
+      "total_tokens": 72391979
+    },
+    {
+      "epoch": 2.8507126781695424,
+      "grad_norm": 0.21530191600322723,
+      "learning_rate": 1.2590728158430431e-06,
+      "loss": 0.4823,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6198,
+      "step": 3800,
+      "tokens_per_second_per_gpu": 4264.41,
+      "total_tokens": 72837687
+    },
+    {
+      "epoch": 2.8694673668417106,
+      "grad_norm": 0.22042331099510193,
+      "learning_rate": 9.602679964744288e-07,
+      "loss": 0.468,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5968,
+      "step": 3825,
+      "tokens_per_second_per_gpu": 4254.73,
+      "total_tokens": 73282095
+    },
+    {
+      "epoch": 2.8882220555138787,
+      "grad_norm": 0.22293563187122345,
+      "learning_rate": 7.017104645146599e-07,
+      "loss": 0.4651,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5922,
+      "step": 3850,
+      "tokens_per_second_per_gpu": 3747.29,
+      "total_tokens": 73724795
+    },
+    {
+      "epoch": 2.9069767441860463,
+      "grad_norm": 0.20620891451835632,
+      "learning_rate": 4.83505291323405e-07,
+      "loss": 0.4598,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.5838,
+      "step": 3875,
+      "tokens_per_second_per_gpu": 4202.85,
+      "total_tokens": 74165185
+    },
+    {
+      "epoch": 2.9257314328582145,
+      "grad_norm": 0.21627213060855865,
+      "learning_rate": 3.0574115006383185e-07,
+      "loss": 0.4713,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6021,
+      "step": 3900,
+      "tokens_per_second_per_gpu": 4239.36,
+      "total_tokens": 74606090
+    },
+    {
+      "epoch": 2.9444861215303826,
+      "grad_norm": 0.22113533318042755,
+      "learning_rate": 1.6849027966816532e-07,
+      "loss": 0.4893,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6312,
+      "step": 3925,
+      "tokens_per_second_per_gpu": 4207.44,
+      "total_tokens": 75045579
+    },
+    {
+      "epoch": 2.9632408102025507,
+      "grad_norm": 0.21696196496486664,
+      "learning_rate": 7.180845548145909e-08,
+      "loss": 0.4804,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6167,
+      "step": 3950,
+      "tokens_per_second_per_gpu": 4152.94,
+      "total_tokens": 75479895
+    },
+    {
+      "epoch": 2.981995498874719,
+      "grad_norm": 0.2239820659160614,
+      "learning_rate": 1.5734966595948308e-08,
+      "loss": 0.4758,
+      "memory/device_reserved (GiB)": 139.06,
+      "memory/max_active (GiB)": 25.53,
+      "memory/max_allocated (GiB)": 25.53,
+      "ppl": 1.6093,
+      "step": 3975,
+      "tokens_per_second_per_gpu": 4224.58,
+      "total_tokens": 75920694
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 3996,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.2526641635157606e+18,
+  "train_batch_size": 5,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-3996/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a75cafa9afe18e1aa48e766453f03add03ceb5dd78011703e6068968ae04eab2
+size 7761