Upload folder using huggingface_hub

README.md

@@ -0,0 +1,158 @@
+---
+library_name: peft
+license: bigcode-openrail-m
+base_model: bigcode/starcoder2-15b
+tags:
+- generated_from_trainer
+datasets:
+- kevinwsbr/vulnfixes-web
+model-index:
+- name: outputs/starcoder-vulnfixes-web
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.8.0.dev0`
+```yaml
+base_model: bigcode/starcoder2-15b
+# Automatically upload checkpoint and final model to HF
+# hub_model_id: username/custom_model_name
+
+load_in_8bit: false
+load_in_4bit: true
+strict: false
+
+datasets:
+  - path: kevinwsbr/vulnfixes-web
+    type: alpaca
+
+
+dataset_prepared_path:
+val_set_size: 0.05
+output_dir: ./outputs/starcoder-vulnfixes-web
+
+adapter: qlora
+lora_model_dir:
+
+sequence_len: 4096
+sample_packing: true
+pad_to_sequence_len: true
+
+lora_r: 32
+lora_alpha: 16
+lora_dropout: 0.05
+lora_target_modules:
+lora_target_linear: true
+lora_fan_in_fan_out:
+
+wandb_project: starcoder
+wandb_entity:
+wandb_watch:
+wandb_run_id:
+wandb_log_model:
+
+gradient_accumulation_steps: 8
+micro_batch_size: 2
+num_epochs: 3
+optimizer: adamw_bnb_8bit
+lr_scheduler: cosine
+learning_rate: 2e-5
+
+train_on_inputs: false
+group_by_length: false
+bf16: auto
+fp16: false
+tf32: false
+
+gradient_checkpointing: true
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+warmup_steps: 20
+evals_per_epoch: 4
+eval_steps:
+eval_table_size:
+saves_per_epoch: 4
+save_steps:
+save_total_limit: 2
+debug:
+deepspeed:
+weight_decay:
+fsdp:
+fsdp_config:
+special_tokens:
+  pad_token: "<|endoftext|>"
+  eos_token: "<|endoftext|>"
+
+```
+
+</details><br>
+
+# outputs/starcoder-vulnfixes-web
+
+This model is a fine-tuned version of [bigcode/starcoder2-15b](https://huggingface.co/bigcode/starcoder2-15b) on the kevinwsbr/vulnfixes-web dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0529
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 16
+- optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 20
+- num_epochs: 3.0
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 0.1499        | 0.0092 | 1    | 0.0645          |
+| 0.1554        | 0.2569 | 28   | 0.0622          |
+| 0.0745        | 0.5138 | 56   | 0.0571          |
+| 0.0616        | 0.7706 | 84   | 0.0559          |
+| 0.0645        | 1.0275 | 112  | 0.0547          |
+| 0.0601        | 1.2844 | 140  | 0.0542          |
+| 0.0688        | 1.5413 | 168  | 0.0537          |
+| 0.0424        | 1.7982 | 196  | 0.0534          |
+| 0.086         | 2.0550 | 224  | 0.0532          |
+| 0.0759        | 2.3119 | 252  | 0.0530          |
+| 0.0583        | 2.5688 | 280  | 0.0529          |
+| 0.1087        | 2.8257 | 308  | 0.0529          |
+
+
+### Framework versions
+
+- PEFT 0.14.0
+- Transformers 4.49.0
+- Pytorch 2.5.1+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

adapter_config.json

@@ -0,0 +1,36 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "bigcode/starcoder2-15b",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_fc",
+    "k_proj",
+    "q_proj",
+    "o_proj",
+    "c_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c90ba47f39c0a86cd549d28055b620e1922d97f151fa40bf696f678998cec60
+size 508623712

checkpoint-308/README.md

@@ -0,0 +1,202 @@
+---
+base_model: bigcode/starcoder2-15b
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

checkpoint-308/adapter_config.json

@@ -0,0 +1,36 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "bigcode/starcoder2-15b",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_fc",
+    "k_proj",
+    "q_proj",
+    "o_proj",
+    "c_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-308/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a9811f43732961fde30393eb326eac6b44f2118cbdc2c73c108fd63a3359a7e
+size 508623712

checkpoint-308/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e544911fbe509ccbc4b3aec76962932edc0d3151ee5da3b0b5d67cf694f1270
+size 258817236

checkpoint-308/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d716d46ab13fe967ada7a71526d8a1189dc582f3667ee74054a1c7be10a5e9a
+size 14244

checkpoint-308/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e55eba8550a28f83d45d944034813f2f9b600be572840059513b77f292bf0b2
+size 1064

checkpoint-308/special_tokens_map.json

@@ -0,0 +1,70 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-308/tokenizer_config.json

@@ -0,0 +1,358 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "37": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

checkpoint-308/trainer_state.json

@@ -0,0 +1,2285 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.8256880733944953,
+  "eval_steps": 28,
+  "global_step": 308,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009174311926605505,
+      "grad_norm": 0.01852019689977169,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1499,
+      "step": 1
+    },
+    {
+      "epoch": 0.009174311926605505,
+      "eval_loss": 0.06453218311071396,
+      "eval_runtime": 43.1581,
+      "eval_samples_per_second": 4.009,
+      "eval_steps_per_second": 2.016,
+      "step": 1
+    },
+    {
+      "epoch": 0.01834862385321101,
+      "grad_norm": 0.02213262766599655,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.1596,
+      "step": 2
+    },
+    {
+      "epoch": 0.027522935779816515,
+      "grad_norm": 0.045894358307123184,
+      "learning_rate": 3e-06,
+      "loss": 0.1794,
+      "step": 3
+    },
+    {
+      "epoch": 0.03669724770642202,
+      "grad_norm": 0.01868272013962269,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.1592,
+      "step": 4
+    },
+    {
+      "epoch": 0.045871559633027525,
+      "grad_norm": 0.017406364902853966,
+      "learning_rate": 5e-06,
+      "loss": 0.1696,
+      "step": 5
+    },
+    {
+      "epoch": 0.05504587155963303,
+      "grad_norm": 0.01861901767551899,
+      "learning_rate": 6e-06,
+      "loss": 0.1438,
+      "step": 6
+    },
+    {
+      "epoch": 0.06422018348623854,
+      "grad_norm": 0.020326999947428703,
+      "learning_rate": 7e-06,
+      "loss": 0.1627,
+      "step": 7
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.02542084828019142,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.1558,
+      "step": 8
+    },
+    {
+      "epoch": 0.08256880733944955,
+      "grad_norm": 0.022425655275583267,
+      "learning_rate": 9e-06,
+      "loss": 0.1517,
+      "step": 9
+    },
+    {
+      "epoch": 0.09174311926605505,
+      "grad_norm": 0.023916274309158325,
+      "learning_rate": 1e-05,
+      "loss": 0.1672,
+      "step": 10
+    },
+    {
+      "epoch": 0.10091743119266056,
+      "grad_norm": 0.022392459213733673,
+      "learning_rate": 1.1000000000000001e-05,
+      "loss": 0.1504,
+      "step": 11
+    },
+    {
+      "epoch": 0.11009174311926606,
+      "grad_norm": 0.02520025707781315,
+      "learning_rate": 1.2e-05,
+      "loss": 0.1749,
+      "step": 12
+    },
+    {
+      "epoch": 0.11926605504587157,
+      "grad_norm": 0.028882062062621117,
+      "learning_rate": 1.3000000000000001e-05,
+      "loss": 0.1705,
+      "step": 13
+    },
+    {
+      "epoch": 0.12844036697247707,
+      "grad_norm": 0.03628265857696533,
+      "learning_rate": 1.4e-05,
+      "loss": 0.166,
+      "step": 14
+    },
+    {
+      "epoch": 0.13761467889908258,
+      "grad_norm": 0.02980518713593483,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 0.1238,
+      "step": 15
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 0.028387300670146942,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.1326,
+      "step": 16
+    },
+    {
+      "epoch": 0.1559633027522936,
+      "grad_norm": 0.03367001935839653,
+      "learning_rate": 1.7e-05,
+      "loss": 0.1347,
+      "step": 17
+    },
+    {
+      "epoch": 0.1651376146788991,
+      "grad_norm": 0.03655758127570152,
+      "learning_rate": 1.8e-05,
+      "loss": 0.1423,
+      "step": 18
+    },
+    {
+      "epoch": 0.1743119266055046,
+      "grad_norm": 0.04000673070549965,
+      "learning_rate": 1.9e-05,
+      "loss": 0.1651,
+      "step": 19
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 0.03844478353857994,
+      "learning_rate": 2e-05,
+      "loss": 0.1649,
+      "step": 20
+    },
+    {
+      "epoch": 0.1926605504587156,
+      "grad_norm": 0.04334944114089012,
+      "learning_rate": 1.99994764125734e-05,
+      "loss": 0.1292,
+      "step": 21
+    },
+    {
+      "epoch": 0.2018348623853211,
+      "grad_norm": 0.04224175587296486,
+      "learning_rate": 1.9997905705122352e-05,
+      "loss": 0.1336,
+      "step": 22
+    },
+    {
+      "epoch": 0.21100917431192662,
+      "grad_norm": 0.04647228121757507,
+      "learning_rate": 1.9995288042127396e-05,
+      "loss": 0.128,
+      "step": 23
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 0.04862922057509422,
+      "learning_rate": 1.9991623697703613e-05,
+      "loss": 0.1611,
+      "step": 24
+    },
+    {
+      "epoch": 0.22935779816513763,
+      "grad_norm": 0.05552718788385391,
+      "learning_rate": 1.998691305557194e-05,
+      "loss": 0.1469,
+      "step": 25
+    },
+    {
+      "epoch": 0.23853211009174313,
+      "grad_norm": 0.04372956603765488,
+      "learning_rate": 1.9981156609018977e-05,
+      "loss": 0.1341,
+      "step": 26
+    },
+    {
+      "epoch": 0.24770642201834864,
+      "grad_norm": 0.04992978647351265,
+      "learning_rate": 1.9974354960845326e-05,
+      "loss": 0.1464,
+      "step": 27
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "grad_norm": 0.04873019829392433,
+      "learning_rate": 1.9966508823302484e-05,
+      "loss": 0.1554,
+      "step": 28
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "eval_loss": 0.06217445060610771,
+      "eval_runtime": 43.3972,
+      "eval_samples_per_second": 3.986,
+      "eval_steps_per_second": 2.005,
+      "step": 28
+    },
+    {
+      "epoch": 0.26605504587155965,
+      "grad_norm": 0.04296933487057686,
+      "learning_rate": 1.9957619018018243e-05,
+      "loss": 0.1231,
+      "step": 29
+    },
+    {
+      "epoch": 0.27522935779816515,
+      "grad_norm": 0.06265883892774582,
+      "learning_rate": 1.9947686475910656e-05,
+      "loss": 0.1292,
+      "step": 30
+    },
+    {
+      "epoch": 0.28440366972477066,
+      "grad_norm": 0.044797539710998535,
+      "learning_rate": 1.9936712237090554e-05,
+      "loss": 0.114,
+      "step": 31
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.05862847715616226,
+      "learning_rate": 1.9924697450752636e-05,
+      "loss": 0.1215,
+      "step": 32
+    },
+    {
+      "epoch": 0.30275229357798167,
+      "grad_norm": 0.04938759654760361,
+      "learning_rate": 1.991164337505511e-05,
+      "loss": 0.0683,
+      "step": 33
+    },
+    {
+      "epoch": 0.3119266055045872,
+      "grad_norm": 0.07766549289226532,
+      "learning_rate": 1.9897551376987948e-05,
+      "loss": 0.1051,
+      "step": 34
+    },
+    {
+      "epoch": 0.3211009174311927,
+      "grad_norm": 0.05474488437175751,
+      "learning_rate": 1.9882422932229765e-05,
+      "loss": 0.082,
+      "step": 35
+    },
+    {
+      "epoch": 0.3302752293577982,
+      "grad_norm": 0.04499150812625885,
+      "learning_rate": 1.9866259624993246e-05,
+      "loss": 0.1135,
+      "step": 36
+    },
+    {
+      "epoch": 0.3394495412844037,
+      "grad_norm": 0.07329924404621124,
+      "learning_rate": 1.9849063147859282e-05,
+      "loss": 0.1082,
+      "step": 37
+    },
+    {
+      "epoch": 0.3486238532110092,
+      "grad_norm": 0.23288355767726898,
+      "learning_rate": 1.983083530159971e-05,
+      "loss": 0.0899,
+      "step": 38
+    },
+    {
+      "epoch": 0.3577981651376147,
+      "grad_norm": 0.06561094522476196,
+      "learning_rate": 1.9811577994988755e-05,
+      "loss": 0.1096,
+      "step": 39
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.052528683096170425,
+      "learning_rate": 1.979129324460314e-05,
+      "loss": 0.107,
+      "step": 40
+    },
+    {
+      "epoch": 0.3761467889908257,
+      "grad_norm": 0.057943329215049744,
+      "learning_rate": 1.9769983174610918e-05,
+      "loss": 0.1121,
+      "step": 41
+    },
+    {
+      "epoch": 0.3853211009174312,
+      "grad_norm": 0.05784667655825615,
+      "learning_rate": 1.974765001654903e-05,
+      "loss": 0.1125,
+      "step": 42
+    },
+    {
+      "epoch": 0.3944954128440367,
+      "grad_norm": 0.04998760297894478,
+      "learning_rate": 1.9724296109089623e-05,
+      "loss": 0.0944,
+      "step": 43
+    },
+    {
+      "epoch": 0.4036697247706422,
+      "grad_norm": 0.056932978332042694,
+      "learning_rate": 1.9699923897795165e-05,
+      "loss": 0.0758,
+      "step": 44
+    },
+    {
+      "epoch": 0.41284403669724773,
+      "grad_norm": 0.05268337205052376,
+      "learning_rate": 1.9674535934862327e-05,
+      "loss": 0.0767,
+      "step": 45
+    },
+    {
+      "epoch": 0.42201834862385323,
+      "grad_norm": 0.04703257977962494,
+      "learning_rate": 1.9648134878854747e-05,
+      "loss": 0.076,
+      "step": 46
+    },
+    {
+      "epoch": 0.43119266055045874,
+      "grad_norm": 0.05632725730538368,
+      "learning_rate": 1.9620723494424627e-05,
+      "loss": 0.1143,
+      "step": 47
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.04887419193983078,
+      "learning_rate": 1.9592304652023208e-05,
+      "loss": 0.096,
+      "step": 48
+    },
+    {
+      "epoch": 0.44954128440366975,
+      "grad_norm": 0.06641880422830582,
+      "learning_rate": 1.9562881327600197e-05,
+      "loss": 0.1108,
+      "step": 49
+    },
+    {
+      "epoch": 0.45871559633027525,
+      "grad_norm": 0.08709923177957535,
+      "learning_rate": 1.9532456602292148e-05,
+      "loss": 0.0987,
+      "step": 50
+    },
+    {
+      "epoch": 0.46788990825688076,
+      "grad_norm": 0.06175887584686279,
+      "learning_rate": 1.950103366209978e-05,
+      "loss": 0.0821,
+      "step": 51
+    },
+    {
+      "epoch": 0.47706422018348627,
+      "grad_norm": 0.05565601587295532,
+      "learning_rate": 1.9468615797554374e-05,
+      "loss": 0.0727,
+      "step": 52
+    },
+    {
+      "epoch": 0.48623853211009177,
+      "grad_norm": 0.13676409423351288,
+      "learning_rate": 1.943520640337318e-05,
+      "loss": 0.0834,
+      "step": 53
+    },
+    {
+      "epoch": 0.4954128440366973,
+      "grad_norm": 0.0817922055721283,
+      "learning_rate": 1.9400808978103948e-05,
+      "loss": 0.0766,
+      "step": 54
+    },
+    {
+      "epoch": 0.5045871559633027,
+      "grad_norm": 0.04707655310630798,
+      "learning_rate": 1.936542712375855e-05,
+      "loss": 0.0753,
+      "step": 55
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.07192892581224442,
+      "learning_rate": 1.9329064545435803e-05,
+      "loss": 0.0745,
+      "step": 56
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "eval_loss": 0.057128190994262695,
+      "eval_runtime": 43.3864,
+      "eval_samples_per_second": 3.987,
+      "eval_steps_per_second": 2.005,
+      "step": 56
+    },
+    {
+      "epoch": 0.5229357798165137,
+      "grad_norm": 0.07009316980838776,
+      "learning_rate": 1.929172505093347e-05,
+      "loss": 0.0696,
+      "step": 57
+    },
+    {
+      "epoch": 0.5321100917431193,
+      "grad_norm": 0.07050078362226486,
+      "learning_rate": 1.9253412550349507e-05,
+      "loss": 0.0846,
+      "step": 58
+    },
+    {
+      "epoch": 0.5412844036697247,
+      "grad_norm": 0.06861168146133423,
+      "learning_rate": 1.9214131055672648e-05,
+      "loss": 0.0659,
+      "step": 59
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.05825705826282501,
+      "learning_rate": 1.917388468036222e-05,
+      "loss": 0.1173,
+      "step": 60
+    },
+    {
+      "epoch": 0.5596330275229358,
+      "grad_norm": 0.11085808277130127,
+      "learning_rate": 1.913267763891745e-05,
+      "loss": 0.0715,
+      "step": 61
+    },
+    {
+      "epoch": 0.5688073394495413,
+      "grad_norm": 0.0637730062007904,
+      "learning_rate": 1.9090514246436085e-05,
+      "loss": 0.0936,
+      "step": 62
+    },
+    {
+      "epoch": 0.5779816513761468,
+      "grad_norm": 0.05786406993865967,
+      "learning_rate": 1.904739891816257e-05,
+      "loss": 0.0777,
+      "step": 63
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.09354288130998611,
+      "learning_rate": 1.9003336169025655e-05,
+      "loss": 0.0913,
+      "step": 64
+    },
+    {
+      "epoch": 0.5963302752293578,
+      "grad_norm": 0.1899336725473404,
+      "learning_rate": 1.8958330613165622e-05,
+      "loss": 0.0761,
+      "step": 65
+    },
+    {
+      "epoch": 0.6055045871559633,
+      "grad_norm": 0.05038751661777496,
+      "learning_rate": 1.891238696345111e-05,
+      "loss": 0.078,
+      "step": 66
+    },
+    {
+      "epoch": 0.6146788990825688,
+      "grad_norm": 0.16387197375297546,
+      "learning_rate": 1.8865510030985588e-05,
+      "loss": 0.0917,
+      "step": 67
+    },
+    {
+      "epoch": 0.6238532110091743,
+      "grad_norm": 0.054418135434389114,
+      "learning_rate": 1.8817704724603536e-05,
+      "loss": 0.0792,
+      "step": 68
+    },
+    {
+      "epoch": 0.6330275229357798,
+      "grad_norm": 0.0637287050485611,
+      "learning_rate": 1.8768976050356428e-05,
+      "loss": 0.0731,
+      "step": 69
+    },
+    {
+      "epoch": 0.6422018348623854,
+      "grad_norm": 0.27382200956344604,
+      "learning_rate": 1.8719329110988487e-05,
+      "loss": 0.0745,
+      "step": 70
+    },
+    {
+      "epoch": 0.6513761467889908,
+      "grad_norm": 0.05016474053263664,
+      "learning_rate": 1.8668769105402366e-05,
+      "loss": 0.0739,
+      "step": 71
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.08013670146465302,
+      "learning_rate": 1.8617301328114704e-05,
+      "loss": 0.1024,
+      "step": 72
+    },
+    {
+      "epoch": 0.6697247706422018,
+      "grad_norm": 0.06279715895652771,
+      "learning_rate": 1.8564931168701713e-05,
+      "loss": 0.0971,
+      "step": 73
+    },
+    {
+      "epoch": 0.6788990825688074,
+      "grad_norm": 0.048747751861810684,
+      "learning_rate": 1.85116641112348e-05,
+      "loss": 0.0657,
+      "step": 74
+    },
+    {
+      "epoch": 0.6880733944954128,
+      "grad_norm": 0.05987577140331268,
+      "learning_rate": 1.845750573370626e-05,
+      "loss": 0.1191,
+      "step": 75
+    },
+    {
+      "epoch": 0.6972477064220184,
+      "grad_norm": 0.08020060509443283,
+      "learning_rate": 1.8402461707445206e-05,
+      "loss": 0.1112,
+      "step": 76
+    },
+    {
+      "epoch": 0.7064220183486238,
+      "grad_norm": 0.05764961615204811,
+      "learning_rate": 1.8346537796523643e-05,
+      "loss": 0.1049,
+      "step": 77
+    },
+    {
+      "epoch": 0.7155963302752294,
+      "grad_norm": 0.08143055438995361,
+      "learning_rate": 1.8289739857152903e-05,
+      "loss": 0.0871,
+      "step": 78
+    },
+    {
+      "epoch": 0.7247706422018348,
+      "grad_norm": 0.08280878514051437,
+      "learning_rate": 1.823207383707036e-05,
+      "loss": 0.0724,
+      "step": 79
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.08486371487379074,
+      "learning_rate": 1.8173545774916628e-05,
+      "loss": 0.068,
+      "step": 80
+    },
+    {
+      "epoch": 0.7431192660550459,
+      "grad_norm": 0.061856675893068314,
+      "learning_rate": 1.8114161799603195e-05,
+      "loss": 0.0786,
+      "step": 81
+    },
+    {
+      "epoch": 0.7522935779816514,
+      "grad_norm": 0.05205192044377327,
+      "learning_rate": 1.8053928129670624e-05,
+      "loss": 0.0755,
+      "step": 82
+    },
+    {
+      "epoch": 0.7614678899082569,
+      "grad_norm": 0.07000340521335602,
+      "learning_rate": 1.7992851072637366e-05,
+      "loss": 0.1239,
+      "step": 83
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "grad_norm": 0.07025006413459778,
+      "learning_rate": 1.793093702433924e-05,
+      "loss": 0.0616,
+      "step": 84
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "eval_loss": 0.055874165147542953,
+      "eval_runtime": 43.2177,
+      "eval_samples_per_second": 4.003,
+      "eval_steps_per_second": 2.013,
+      "step": 84
+    },
+    {
+      "epoch": 0.7798165137614679,
+      "grad_norm": 0.06147678196430206,
+      "learning_rate": 1.7868192468259686e-05,
+      "loss": 0.0917,
+      "step": 85
+    },
+    {
+      "epoch": 0.7889908256880734,
+      "grad_norm": 0.045870471745729446,
+      "learning_rate": 1.7804623974850844e-05,
+      "loss": 0.092,
+      "step": 86
+    },
+    {
+      "epoch": 0.7981651376146789,
+      "grad_norm": 0.07622863352298737,
+      "learning_rate": 1.7740238200845485e-05,
+      "loss": 0.0983,
+      "step": 87
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.08215321600437164,
+      "learning_rate": 1.7675041888559952e-05,
+      "loss": 0.0971,
+      "step": 88
+    },
+    {
+      "epoch": 0.8165137614678899,
+      "grad_norm": 0.06286073476076126,
+      "learning_rate": 1.7609041865188122e-05,
+      "loss": 0.0876,
+      "step": 89
+    },
+    {
+      "epoch": 0.8256880733944955,
+      "grad_norm": 0.06399139016866684,
+      "learning_rate": 1.754224504208647e-05,
+      "loss": 0.1166,
+      "step": 90
+    },
+    {
+      "epoch": 0.8348623853211009,
+      "grad_norm": 0.07115256041288376,
+      "learning_rate": 1.7474658414050344e-05,
+      "loss": 0.119,
+      "step": 91
+    },
+    {
+      "epoch": 0.8440366972477065,
+      "grad_norm": 0.04867429658770561,
+      "learning_rate": 1.7406289058581466e-05,
+      "loss": 0.0609,
+      "step": 92
+    },
+    {
+      "epoch": 0.8532110091743119,
+      "grad_norm": 0.1833045333623886,
+      "learning_rate": 1.7337144135146818e-05,
+      "loss": 0.1247,
+      "step": 93
+    },
+    {
+      "epoch": 0.8623853211009175,
+      "grad_norm": 0.07023169845342636,
+      "learning_rate": 1.7267230884428905e-05,
+      "loss": 0.1271,
+      "step": 94
+    },
+    {
+      "epoch": 0.8715596330275229,
+      "grad_norm": 0.044201672077178955,
+      "learning_rate": 1.719655662756753e-05,
+      "loss": 0.0661,
+      "step": 95
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.05139552056789398,
+      "learning_rate": 1.7125128765393157e-05,
+      "loss": 0.1065,
+      "step": 96
+    },
+    {
+      "epoch": 0.8899082568807339,
+      "grad_norm": 0.06401531398296356,
+      "learning_rate": 1.705295477765188e-05,
+      "loss": 0.1006,
+      "step": 97
+    },
+    {
+      "epoch": 0.8990825688073395,
+      "grad_norm": 0.04898101091384888,
+      "learning_rate": 1.6980042222222216e-05,
+      "loss": 0.0522,
+      "step": 98
+    },
+    {
+      "epoch": 0.908256880733945,
+      "grad_norm": 0.053434859961271286,
+      "learning_rate": 1.690639873432361e-05,
+      "loss": 0.1214,
+      "step": 99
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.058289702981710434,
+      "learning_rate": 1.683203202571692e-05,
+      "loss": 0.0547,
+      "step": 100
+    },
+    {
+      "epoch": 0.926605504587156,
+      "grad_norm": 0.06972479820251465,
+      "learning_rate": 1.6756949883896874e-05,
+      "loss": 0.0867,
+      "step": 101
+    },
+    {
+      "epoch": 0.9357798165137615,
+      "grad_norm": 0.05255963280797005,
+      "learning_rate": 1.668116017127655e-05,
+      "loss": 0.0685,
+      "step": 102
+    },
+    {
+      "epoch": 0.944954128440367,
+      "grad_norm": 0.04849875345826149,
+      "learning_rate": 1.6604670824364067e-05,
+      "loss": 0.074,
+      "step": 103
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.08504751324653625,
+      "learning_rate": 1.652748985293149e-05,
+      "loss": 0.0901,
+      "step": 104
+    },
+    {
+      "epoch": 0.963302752293578,
+      "grad_norm": 0.07853557169437408,
+      "learning_rate": 1.6449625339176056e-05,
+      "loss": 0.0663,
+      "step": 105
+    },
+    {
+      "epoch": 0.9724770642201835,
+      "grad_norm": 0.0786280408501625,
+      "learning_rate": 1.6371085436873847e-05,
+      "loss": 0.0936,
+      "step": 106
+    },
+    {
+      "epoch": 0.981651376146789,
+      "grad_norm": 0.09738124907016754,
+      "learning_rate": 1.6291878370525925e-05,
+      "loss": 0.1001,
+      "step": 107
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": 0.061633653938770294,
+      "learning_rate": 1.6212012434497103e-05,
+      "loss": 0.0728,
+      "step": 108
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.06726932525634766,
+      "learning_rate": 1.6131495992147363e-05,
+      "loss": 0.0837,
+      "step": 109
+    },
+    {
+      "epoch": 1.0091743119266054,
+      "grad_norm": 0.05419926717877388,
+      "learning_rate": 1.605033747495607e-05,
+      "loss": 0.0841,
+      "step": 110
+    },
+    {
+      "epoch": 1.018348623853211,
+      "grad_norm": 0.056966476142406464,
+      "learning_rate": 1.596854538163906e-05,
+      "loss": 0.0774,
+      "step": 111
+    },
+    {
+      "epoch": 1.0275229357798166,
+      "grad_norm": 0.05710803344845772,
+      "learning_rate": 1.5886128277258665e-05,
+      "loss": 0.0645,
+      "step": 112
+    },
+    {
+      "epoch": 1.0275229357798166,
+      "eval_loss": 0.05474008619785309,
+      "eval_runtime": 43.3773,
+      "eval_samples_per_second": 3.988,
+      "eval_steps_per_second": 2.006,
+      "step": 112
+    },
+    {
+      "epoch": 1.036697247706422,
+      "grad_norm": 0.05264132842421532,
+      "learning_rate": 1.58030947923268e-05,
+      "loss": 0.0615,
+      "step": 113
+    },
+    {
+      "epoch": 1.0458715596330275,
+      "grad_norm": 0.11162517964839935,
+      "learning_rate": 1.571945362190121e-05,
+      "loss": 0.13,
+      "step": 114
+    },
+    {
+      "epoch": 1.0550458715596331,
+      "grad_norm": 0.05422775819897652,
+      "learning_rate": 1.563521352467493e-05,
+      "loss": 0.0677,
+      "step": 115
+    },
+    {
+      "epoch": 1.0642201834862386,
+      "grad_norm": 0.08082108199596405,
+      "learning_rate": 1.55503833220591e-05,
+      "loss": 0.0958,
+      "step": 116
+    },
+    {
+      "epoch": 1.073394495412844,
+      "grad_norm": 0.0667729526758194,
+      "learning_rate": 1.546497189725922e-05,
+      "loss": 0.1132,
+      "step": 117
+    },
+    {
+      "epoch": 1.0825688073394495,
+      "grad_norm": 0.06297166645526886,
+      "learning_rate": 1.5378988194344913e-05,
+      "loss": 0.0646,
+      "step": 118
+    },
+    {
+      "epoch": 1.091743119266055,
+      "grad_norm": 0.05654435604810715,
+      "learning_rate": 1.5292441217313324e-05,
+      "loss": 0.1083,
+      "step": 119
+    },
+    {
+      "epoch": 1.1009174311926606,
+      "grad_norm": 0.054773300886154175,
+      "learning_rate": 1.5205340029146256e-05,
+      "loss": 0.0833,
+      "step": 120
+    },
+    {
+      "epoch": 1.110091743119266,
+      "grad_norm": 0.0510234571993351,
+      "learning_rate": 1.5117693750861096e-05,
+      "loss": 0.0792,
+      "step": 121
+    },
+    {
+      "epoch": 1.1192660550458715,
+      "grad_norm": 0.054006725549697876,
+      "learning_rate": 1.5029511560555707e-05,
+      "loss": 0.1066,
+      "step": 122
+    },
+    {
+      "epoch": 1.1284403669724772,
+      "grad_norm": 0.06665553152561188,
+      "learning_rate": 1.4940802692447306e-05,
+      "loss": 0.0584,
+      "step": 123
+    },
+    {
+      "epoch": 1.1376146788990826,
+      "grad_norm": 0.105413518846035,
+      "learning_rate": 1.4851576435905489e-05,
+      "loss": 0.0907,
+      "step": 124
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.0626402273774147,
+      "learning_rate": 1.4761842134479463e-05,
+      "loss": 0.0544,
+      "step": 125
+    },
+    {
+      "epoch": 1.1559633027522935,
+      "grad_norm": 0.06280255317687988,
+      "learning_rate": 1.4671609184919622e-05,
+      "loss": 0.0487,
+      "step": 126
+    },
+    {
+      "epoch": 1.165137614678899,
+      "grad_norm": 0.07118494808673859,
+      "learning_rate": 1.4580887036193539e-05,
+      "loss": 0.0797,
+      "step": 127
+    },
+    {
+      "epoch": 1.1743119266055047,
+      "grad_norm": 0.0436442606151104,
+      "learning_rate": 1.4489685188496488e-05,
+      "loss": 0.0568,
+      "step": 128
+    },
+    {
+      "epoch": 1.18348623853211,
+      "grad_norm": 0.06852032989263535,
+      "learning_rate": 1.4398013192256615e-05,
+      "loss": 0.1099,
+      "step": 129
+    },
+    {
+      "epoch": 1.1926605504587156,
+      "grad_norm": 0.05673675611615181,
+      "learning_rate": 1.4305880647134847e-05,
+      "loss": 0.0649,
+      "step": 130
+    },
+    {
+      "epoch": 1.2018348623853212,
+      "grad_norm": 0.05407770350575447,
+      "learning_rate": 1.4213297201019618e-05,
+      "loss": 0.0582,
+      "step": 131
+    },
+    {
+      "epoch": 1.2110091743119267,
+      "grad_norm": 0.06976728141307831,
+      "learning_rate": 1.4120272549016591e-05,
+      "loss": 0.0744,
+      "step": 132
+    },
+    {
+      "epoch": 1.2201834862385321,
+      "grad_norm": 0.044198598712682724,
+      "learning_rate": 1.40268164324334e-05,
+      "loss": 0.0456,
+      "step": 133
+    },
+    {
+      "epoch": 1.2293577981651376,
+      "grad_norm": 0.052981797605752945,
+      "learning_rate": 1.3932938637759555e-05,
+      "loss": 0.067,
+      "step": 134
+    },
+    {
+      "epoch": 1.238532110091743,
+      "grad_norm": 0.051931336522102356,
+      "learning_rate": 1.3838648995641645e-05,
+      "loss": 0.0662,
+      "step": 135
+    },
+    {
+      "epoch": 1.2477064220183487,
+      "grad_norm": 0.06196126341819763,
+      "learning_rate": 1.3743957379853885e-05,
+      "loss": 0.0742,
+      "step": 136
+    },
+    {
+      "epoch": 1.2568807339449541,
+      "grad_norm": 0.0772649347782135,
+      "learning_rate": 1.3648873706264159e-05,
+      "loss": 0.1016,
+      "step": 137
+    },
+    {
+      "epoch": 1.2660550458715596,
+      "grad_norm": 0.06751381605863571,
+      "learning_rate": 1.3553407931795662e-05,
+      "loss": 0.0871,
+      "step": 138
+    },
+    {
+      "epoch": 1.2752293577981653,
+      "grad_norm": 0.045472558587789536,
+      "learning_rate": 1.3457570053384225e-05,
+      "loss": 0.0548,
+      "step": 139
+    },
+    {
+      "epoch": 1.2844036697247707,
+      "grad_norm": 0.061834368854761124,
+      "learning_rate": 1.3361370106931486e-05,
+      "loss": 0.0601,
+      "step": 140
+    },
+    {
+      "epoch": 1.2844036697247707,
+      "eval_loss": 0.05419979989528656,
+      "eval_runtime": 43.2558,
+      "eval_samples_per_second": 3.999,
+      "eval_steps_per_second": 2.011,
+      "step": 140
+    },
+    {
+      "epoch": 1.2935779816513762,
+      "grad_norm": 0.062169574201107025,
+      "learning_rate": 1.3264818166253917e-05,
+      "loss": 0.0899,
+      "step": 141
+    },
+    {
+      "epoch": 1.3027522935779816,
+      "grad_norm": 0.051288578659296036,
+      "learning_rate": 1.3167924342027947e-05,
+      "loss": 0.0758,
+      "step": 142
+    },
+    {
+      "epoch": 1.311926605504587,
+      "grad_norm": 0.052053675055503845,
+      "learning_rate": 1.3070698780731194e-05,
+      "loss": 0.0989,
+      "step": 143
+    },
+    {
+      "epoch": 1.3211009174311927,
+      "grad_norm": 0.05368887633085251,
+      "learning_rate": 1.2973151663579948e-05,
+      "loss": 0.0599,
+      "step": 144
+    },
+    {
+      "epoch": 1.3302752293577982,
+      "grad_norm": 0.06439553946256638,
+      "learning_rate": 1.2875293205463018e-05,
+      "loss": 0.0808,
+      "step": 145
+    },
+    {
+      "epoch": 1.3394495412844036,
+      "grad_norm": 0.05169299989938736,
+      "learning_rate": 1.277713365387205e-05,
+      "loss": 0.0668,
+      "step": 146
+    },
+    {
+      "epoch": 1.3486238532110093,
+      "grad_norm": 0.0563080795109272,
+      "learning_rate": 1.2678683287828451e-05,
+      "loss": 0.0888,
+      "step": 147
+    },
+    {
+      "epoch": 1.3577981651376148,
+      "grad_norm": 0.06615187972784042,
+      "learning_rate": 1.257995241680698e-05,
+      "loss": 0.123,
+      "step": 148
+    },
+    {
+      "epoch": 1.3669724770642202,
+      "grad_norm": 0.0562995970249176,
+      "learning_rate": 1.2480951379656175e-05,
+      "loss": 0.0709,
+      "step": 149
+    },
+    {
+      "epoch": 1.3761467889908257,
+      "grad_norm": 0.06163397431373596,
+      "learning_rate": 1.2381690543515692e-05,
+      "loss": 0.0663,
+      "step": 150
+    },
+    {
+      "epoch": 1.385321100917431,
+      "grad_norm": 0.053988683968782425,
+      "learning_rate": 1.2282180302730683e-05,
+      "loss": 0.0812,
+      "step": 151
+    },
+    {
+      "epoch": 1.3944954128440368,
+      "grad_norm": 0.062011417001485825,
+      "learning_rate": 1.2182431077763317e-05,
+      "loss": 0.0821,
+      "step": 152
+    },
+    {
+      "epoch": 1.4036697247706422,
+      "grad_norm": 0.054933421313762665,
+      "learning_rate": 1.2082453314101607e-05,
+      "loss": 0.0726,
+      "step": 153
+    },
+    {
+      "epoch": 1.4128440366972477,
+      "grad_norm": 0.0531560480594635,
+      "learning_rate": 1.1982257481165547e-05,
+      "loss": 0.052,
+      "step": 154
+    },
+    {
+      "epoch": 1.4220183486238533,
+      "grad_norm": 0.0668332502245903,
+      "learning_rate": 1.1881854071210805e-05,
+      "loss": 0.0758,
+      "step": 155
+    },
+    {
+      "epoch": 1.4311926605504588,
+      "grad_norm": 0.06946220993995667,
+      "learning_rate": 1.1781253598229982e-05,
+      "loss": 0.0702,
+      "step": 156
+    },
+    {
+      "epoch": 1.4403669724770642,
+      "grad_norm": 0.06085884943604469,
+      "learning_rate": 1.1680466596851635e-05,
+      "loss": 0.0727,
+      "step": 157
+    },
+    {
+      "epoch": 1.4495412844036697,
+      "grad_norm": 0.04978602007031441,
+      "learning_rate": 1.1579503621237102e-05,
+      "loss": 0.077,
+      "step": 158
+    },
+    {
+      "epoch": 1.4587155963302751,
+      "grad_norm": 0.05822195112705231,
+      "learning_rate": 1.1478375243975298e-05,
+      "loss": 0.0769,
+      "step": 159
+    },
+    {
+      "epoch": 1.4678899082568808,
+      "grad_norm": 0.06906376779079437,
+      "learning_rate": 1.1377092054975586e-05,
+      "loss": 0.094,
+      "step": 160
+    },
+    {
+      "epoch": 1.4770642201834863,
+      "grad_norm": 0.0629039779305458,
+      "learning_rate": 1.1275664660358818e-05,
+      "loss": 0.0491,
+      "step": 161
+    },
+    {
+      "epoch": 1.4862385321100917,
+      "grad_norm": 0.05990944802761078,
+      "learning_rate": 1.1174103681346711e-05,
+      "loss": 0.0514,
+      "step": 162
+    },
+    {
+      "epoch": 1.4954128440366974,
+      "grad_norm": 0.055665481835603714,
+      "learning_rate": 1.1072419753149585e-05,
+      "loss": 0.0652,
+      "step": 163
+    },
+    {
+      "epoch": 1.5045871559633026,
+      "grad_norm": 0.04991630092263222,
+      "learning_rate": 1.0970623523852699e-05,
+      "loss": 0.0576,
+      "step": 164
+    },
+    {
+      "epoch": 1.5137614678899083,
+      "grad_norm": 0.052877284586429596,
+      "learning_rate": 1.0868725653301206e-05,
+      "loss": 0.075,
+      "step": 165
+    },
+    {
+      "epoch": 1.5229357798165137,
+      "grad_norm": 0.0749133750796318,
+      "learning_rate": 1.0766736811983864e-05,
+      "loss": 0.092,
+      "step": 166
+    },
+    {
+      "epoch": 1.5321100917431192,
+      "grad_norm": 0.050854723900556564,
+      "learning_rate": 1.066466767991567e-05,
+      "loss": 0.0282,
+      "step": 167
+    },
+    {
+      "epoch": 1.5412844036697249,
+      "grad_norm": 0.05233798176050186,
+      "learning_rate": 1.0562528945519463e-05,
+      "loss": 0.0688,
+      "step": 168
+    },
+    {
+      "epoch": 1.5412844036697249,
+      "eval_loss": 0.05367153137922287,
+      "eval_runtime": 43.3753,
+      "eval_samples_per_second": 3.988,
+      "eval_steps_per_second": 2.006,
+      "step": 168
+    },
+    {
+      "epoch": 1.5504587155963303,
+      "grad_norm": 0.06258448958396912,
+      "learning_rate": 1.0460331304506658e-05,
+      "loss": 0.0601,
+      "step": 169
+    },
+    {
+      "epoch": 1.5596330275229358,
+      "grad_norm": 0.05399218574166298,
+      "learning_rate": 1.0358085458757233e-05,
+      "loss": 0.0642,
+      "step": 170
+    },
+    {
+      "epoch": 1.5688073394495414,
+      "grad_norm": 0.052848368883132935,
+      "learning_rate": 1.0255802115199034e-05,
+      "loss": 0.054,
+      "step": 171
+    },
+    {
+      "epoch": 1.5779816513761467,
+      "grad_norm": 0.054908327758312225,
+      "learning_rate": 1.0153491984686595e-05,
+      "loss": 0.069,
+      "step": 172
+    },
+    {
+      "epoch": 1.5871559633027523,
+      "grad_norm": 0.08597759157419205,
+      "learning_rate": 1.0051165780879503e-05,
+      "loss": 0.0596,
+      "step": 173
+    },
+    {
+      "epoch": 1.5963302752293578,
+      "grad_norm": 0.06400232017040253,
+      "learning_rate": 9.9488342191205e-06,
+      "loss": 0.0959,
+      "step": 174
+    },
+    {
+      "epoch": 1.6055045871559632,
+      "grad_norm": 0.06301407516002655,
+      "learning_rate": 9.846508015313407e-06,
+      "loss": 0.0863,
+      "step": 175
+    },
+    {
+      "epoch": 1.614678899082569,
+      "grad_norm": 0.09353712201118469,
+      "learning_rate": 9.744197884800968e-06,
+      "loss": 0.0835,
+      "step": 176
+    },
+    {
+      "epoch": 1.6238532110091743,
+      "grad_norm": 0.14486946165561676,
+      "learning_rate": 9.64191454124277e-06,
+      "loss": 0.0786,
+      "step": 177
+    },
+    {
+      "epoch": 1.6330275229357798,
+      "grad_norm": 0.06457175314426422,
+      "learning_rate": 9.539668695493344e-06,
+      "loss": 0.0752,
+      "step": 178
+    },
+    {
+      "epoch": 1.6422018348623855,
+      "grad_norm": 0.059133633971214294,
+      "learning_rate": 9.43747105448054e-06,
+      "loss": 0.0747,
+      "step": 179
+    },
+    {
+      "epoch": 1.6513761467889907,
+      "grad_norm": 0.04722464829683304,
+      "learning_rate": 9.335332320084331e-06,
+      "loss": 0.0503,
+      "step": 180
+    },
+    {
+      "epoch": 1.6605504587155964,
+      "grad_norm": 0.06446046382188797,
+      "learning_rate": 9.233263188016138e-06,
+      "loss": 0.0794,
+      "step": 181
+    },
+    {
+      "epoch": 1.6697247706422018,
+      "grad_norm": 0.05856352299451828,
+      "learning_rate": 9.131274346698797e-06,
+      "loss": 0.0917,
+      "step": 182
+    },
+    {
+      "epoch": 1.6788990825688073,
+      "grad_norm": 0.050151705741882324,
+      "learning_rate": 9.029376476147303e-06,
+      "loss": 0.0534,
+      "step": 183
+    },
+    {
+      "epoch": 1.688073394495413,
+      "grad_norm": 0.11409275978803635,
+      "learning_rate": 8.927580246850418e-06,
+      "loss": 0.0579,
+      "step": 184
+    },
+    {
+      "epoch": 1.6972477064220184,
+      "grad_norm": 0.04147953912615776,
+      "learning_rate": 8.825896318653294e-06,
+      "loss": 0.0596,
+      "step": 185
+    },
+    {
+      "epoch": 1.7064220183486238,
+      "grad_norm": 0.06895549595355988,
+      "learning_rate": 8.724335339641185e-06,
+      "loss": 0.1267,
+      "step": 186
+    },
+    {
+      "epoch": 1.7155963302752295,
+      "grad_norm": 0.07597438991069794,
+      "learning_rate": 8.622907945024418e-06,
+      "loss": 0.0672,
+      "step": 187
+    },
+    {
+      "epoch": 1.7247706422018347,
+      "grad_norm": 0.0447760745882988,
+      "learning_rate": 8.521624756024706e-06,
+      "loss": 0.0619,
+      "step": 188
+    },
+    {
+      "epoch": 1.7339449541284404,
+      "grad_norm": 0.054984625428915024,
+      "learning_rate": 8.420496378762901e-06,
+      "loss": 0.0665,
+      "step": 189
+    },
+    {
+      "epoch": 1.7431192660550459,
+      "grad_norm": 0.06113699823617935,
+      "learning_rate": 8.319533403148368e-06,
+      "loss": 0.0755,
+      "step": 190
+    },
+    {
+      "epoch": 1.7522935779816513,
+      "grad_norm": 0.056827329099178314,
+      "learning_rate": 8.218746401770021e-06,
+      "loss": 0.0674,
+      "step": 191
+    },
+    {
+      "epoch": 1.761467889908257,
+      "grad_norm": 0.06809786707162857,
+      "learning_rate": 8.118145928789198e-06,
+      "loss": 0.0595,
+      "step": 192
+    },
+    {
+      "epoch": 1.7706422018348624,
+      "grad_norm": 0.05617869272828102,
+      "learning_rate": 8.017742518834454e-06,
+      "loss": 0.0823,
+      "step": 193
+    },
+    {
+      "epoch": 1.7798165137614679,
+      "grad_norm": 0.05785394459962845,
+      "learning_rate": 7.917546685898393e-06,
+      "loss": 0.099,
+      "step": 194
+    },
+    {
+      "epoch": 1.7889908256880735,
+      "grad_norm": 0.04695114120841026,
+      "learning_rate": 7.817568922236683e-06,
+      "loss": 0.0559,
+      "step": 195
+    },
+    {
+      "epoch": 1.7981651376146788,
+      "grad_norm": 0.04976266250014305,
+      "learning_rate": 7.717819697269322e-06,
+      "loss": 0.0424,
+      "step": 196
+    },
+    {
+      "epoch": 1.7981651376146788,
+      "eval_loss": 0.05340273305773735,
+      "eval_runtime": 43.3761,
+      "eval_samples_per_second": 3.988,
+      "eval_steps_per_second": 2.006,
+      "step": 196
+    },
+    {
+      "epoch": 1.8073394495412844,
+      "grad_norm": 0.04787492752075195,
+      "learning_rate": 7.618309456484309e-06,
+      "loss": 0.0439,
+      "step": 197
+    },
+    {
+      "epoch": 1.81651376146789,
+      "grad_norm": 0.048712193965911865,
+      "learning_rate": 7.519048620343825e-06,
+      "loss": 0.0714,
+      "step": 198
+    },
+    {
+      "epoch": 1.8256880733944953,
+      "grad_norm": 0.06686241924762726,
+      "learning_rate": 7.42004758319302e-06,
+      "loss": 0.0746,
+      "step": 199
+    },
+    {
+      "epoch": 1.834862385321101,
+      "grad_norm": 0.06182721257209778,
+      "learning_rate": 7.3213167121715514e-06,
+      "loss": 0.068,
+      "step": 200
+    },
+    {
+      "epoch": 1.8440366972477065,
+      "grad_norm": 0.05911610275506973,
+      "learning_rate": 7.222866346127952e-06,
+      "loss": 0.061,
+      "step": 201
+    },
+    {
+      "epoch": 1.853211009174312,
+      "grad_norm": 0.048602789640426636,
+      "learning_rate": 7.124706794536984e-06,
+      "loss": 0.0545,
+      "step": 202
+    },
+    {
+      "epoch": 1.8623853211009176,
+      "grad_norm": 0.09133545309305191,
+      "learning_rate": 7.026848336420053e-06,
+      "loss": 0.0579,
+      "step": 203
+    },
+    {
+      "epoch": 1.8715596330275228,
+      "grad_norm": 0.05401468276977539,
+      "learning_rate": 6.929301219268806e-06,
+      "loss": 0.0498,
+      "step": 204
+    },
+    {
+      "epoch": 1.8807339449541285,
+      "grad_norm": 0.057028841227293015,
+      "learning_rate": 6.8320756579720545e-06,
+      "loss": 0.1163,
+      "step": 205
+    },
+    {
+      "epoch": 1.889908256880734,
+      "grad_norm": 0.06496277451515198,
+      "learning_rate": 6.735181833746087e-06,
+      "loss": 0.0687,
+      "step": 206
+    },
+    {
+      "epoch": 1.8990825688073394,
+      "grad_norm": 0.04509355500340462,
+      "learning_rate": 6.638629893068516e-06,
+      "loss": 0.0833,
+      "step": 207
+    },
+    {
+      "epoch": 1.908256880733945,
+      "grad_norm": 0.05287083983421326,
+      "learning_rate": 6.542429946615774e-06,
+      "loss": 0.0506,
+      "step": 208
+    },
+    {
+      "epoch": 1.9174311926605505,
+      "grad_norm": 0.0586075522005558,
+      "learning_rate": 6.446592068204341e-06,
+      "loss": 0.0747,
+      "step": 209
+    },
+    {
+      "epoch": 1.926605504587156,
+      "grad_norm": 0.05395258963108063,
+      "learning_rate": 6.351126293735843e-06,
+      "loss": 0.0703,
+      "step": 210
+    },
+    {
+      "epoch": 1.9357798165137616,
+      "grad_norm": 0.04344907030463219,
+      "learning_rate": 6.256042620146119e-06,
+      "loss": 0.0445,
+      "step": 211
+    },
+    {
+      "epoch": 1.9449541284403669,
+      "grad_norm": 0.047881245613098145,
+      "learning_rate": 6.16135100435836e-06,
+      "loss": 0.063,
+      "step": 212
+    },
+    {
+      "epoch": 1.9541284403669725,
+      "grad_norm": 0.0735430121421814,
+      "learning_rate": 6.06706136224045e-06,
+      "loss": 0.0936,
+      "step": 213
+    },
+    {
+      "epoch": 1.963302752293578,
+      "grad_norm": 0.0707215815782547,
+      "learning_rate": 5.973183567566605e-06,
+      "loss": 0.0906,
+      "step": 214
+    },
+    {
+      "epoch": 1.9724770642201834,
+      "grad_norm": 0.05329213663935661,
+      "learning_rate": 5.879727450983412e-06,
+      "loss": 0.0665,
+      "step": 215
+    },
+    {
+      "epoch": 1.981651376146789,
+      "grad_norm": 0.04940500482916832,
+      "learning_rate": 5.786702798980388e-06,
+      "loss": 0.0649,
+      "step": 216
+    },
+    {
+      "epoch": 1.9908256880733946,
+      "grad_norm": 0.06716382503509521,
+      "learning_rate": 5.69411935286516e-06,
+      "loss": 0.0882,
+      "step": 217
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.06293365359306335,
+      "learning_rate": 5.601986807743388e-06,
+      "loss": 0.0649,
+      "step": 218
+    },
+    {
+      "epoch": 2.0091743119266057,
+      "grad_norm": 0.06269426643848419,
+      "learning_rate": 5.51031481150352e-06,
+      "loss": 0.0757,
+      "step": 219
+    },
+    {
+      "epoch": 2.018348623853211,
+      "grad_norm": 0.05706481635570526,
+      "learning_rate": 5.419112963806468e-06,
+      "loss": 0.0519,
+      "step": 220
+    },
+    {
+      "epoch": 2.0275229357798166,
+      "grad_norm": 0.05423853173851967,
+      "learning_rate": 5.328390815080381e-06,
+      "loss": 0.0828,
+      "step": 221
+    },
+    {
+      "epoch": 2.036697247706422,
+      "grad_norm": 0.04003476724028587,
+      "learning_rate": 5.238157865520539e-06,
+      "loss": 0.0431,
+      "step": 222
+    },
+    {
+      "epoch": 2.0458715596330275,
+      "grad_norm": 0.047263018786907196,
+      "learning_rate": 5.148423564094517e-06,
+      "loss": 0.0498,
+      "step": 223
+    },
+    {
+      "epoch": 2.055045871559633,
+      "grad_norm": 0.06567783653736115,
+      "learning_rate": 5.059197307552698e-06,
+      "loss": 0.086,
+      "step": 224
+    },
+    {
+      "epoch": 2.055045871559633,
+      "eval_loss": 0.05316643789410591,
+      "eval_runtime": 43.3865,
+      "eval_samples_per_second": 3.987,
+      "eval_steps_per_second": 2.005,
+      "step": 224
+    },
+    {
+      "epoch": 2.0642201834862384,
+      "grad_norm": 0.04639436677098274,
+      "learning_rate": 4.970488439444296e-06,
+      "loss": 0.0589,
+      "step": 225
+    },
+    {
+      "epoch": 2.073394495412844,
+      "grad_norm": 0.06990166008472443,
+      "learning_rate": 4.882306249138909e-06,
+      "loss": 0.0491,
+      "step": 226
+    },
+    {
+      "epoch": 2.0825688073394497,
+      "grad_norm": 0.05951413884758949,
+      "learning_rate": 4.7946599708537485e-06,
+      "loss": 0.0736,
+      "step": 227
+    },
+    {
+      "epoch": 2.091743119266055,
+      "grad_norm": 0.04998117685317993,
+      "learning_rate": 4.707558782686677e-06,
+      "loss": 0.073,
+      "step": 228
+    },
+    {
+      "epoch": 2.1009174311926606,
+      "grad_norm": 0.05194695666432381,
+      "learning_rate": 4.621011805655093e-06,
+      "loss": 0.0797,
+      "step": 229
+    },
+    {
+      "epoch": 2.1100917431192663,
+      "grad_norm": 0.058086711913347244,
+      "learning_rate": 4.535028102740785e-06,
+      "loss": 0.0612,
+      "step": 230
+    },
+    {
+      "epoch": 2.1192660550458715,
+      "grad_norm": 0.05054466798901558,
+      "learning_rate": 4.449616677940904e-06,
+      "loss": 0.0623,
+      "step": 231
+    },
+    {
+      "epoch": 2.128440366972477,
+      "grad_norm": 0.05400394648313522,
+      "learning_rate": 4.364786475325072e-06,
+      "loss": 0.0778,
+      "step": 232
+    },
+    {
+      "epoch": 2.1376146788990824,
+      "grad_norm": 0.049106135964393616,
+      "learning_rate": 4.280546378098792e-06,
+      "loss": 0.0673,
+      "step": 233
+    },
+    {
+      "epoch": 2.146788990825688,
+      "grad_norm": 0.05438579246401787,
+      "learning_rate": 4.196905207673201e-06,
+      "loss": 0.071,
+      "step": 234
+    },
+    {
+      "epoch": 2.1559633027522938,
+      "grad_norm": 0.05540682002902031,
+      "learning_rate": 4.113871722741337e-06,
+      "loss": 0.0691,
+      "step": 235
+    },
+    {
+      "epoch": 2.165137614678899,
+      "grad_norm": 0.07121812552213669,
+      "learning_rate": 4.031454618360945e-06,
+      "loss": 0.0859,
+      "step": 236
+    },
+    {
+      "epoch": 2.1743119266055047,
+      "grad_norm": 0.06990870833396912,
+      "learning_rate": 3.949662525043935e-06,
+      "loss": 0.0808,
+      "step": 237
+    },
+    {
+      "epoch": 2.18348623853211,
+      "grad_norm": 0.04688805714249611,
+      "learning_rate": 3.868504007852641e-06,
+      "loss": 0.0552,
+      "step": 238
+    },
+    {
+      "epoch": 2.1926605504587156,
+      "grad_norm": 0.05368739366531372,
+      "learning_rate": 3.7879875655029018e-06,
+      "loss": 0.0708,
+      "step": 239
+    },
+    {
+      "epoch": 2.2018348623853212,
+      "grad_norm": 0.05669724941253662,
+      "learning_rate": 3.7081216294740773e-06,
+      "loss": 0.0714,
+      "step": 240
+    },
+    {
+      "epoch": 2.2110091743119265,
+      "grad_norm": 0.07314234972000122,
+      "learning_rate": 3.628914563126156e-06,
+      "loss": 0.0842,
+      "step": 241
+    },
+    {
+      "epoch": 2.220183486238532,
+      "grad_norm": 0.06005195155739784,
+      "learning_rate": 3.5503746608239487e-06,
+      "loss": 0.0637,
+      "step": 242
+    },
+    {
+      "epoch": 2.229357798165138,
+      "grad_norm": 0.05186507850885391,
+      "learning_rate": 3.472510147068515e-06,
+      "loss": 0.0606,
+      "step": 243
+    },
+    {
+      "epoch": 2.238532110091743,
+      "grad_norm": 0.058381304144859314,
+      "learning_rate": 3.3953291756359354e-06,
+      "loss": 0.0626,
+      "step": 244
+    },
+    {
+      "epoch": 2.2477064220183487,
+      "grad_norm": 0.05521458014845848,
+      "learning_rate": 3.3188398287234504e-06,
+      "loss": 0.0628,
+      "step": 245
+    },
+    {
+      "epoch": 2.2568807339449544,
+      "grad_norm": 0.05869848653674126,
+      "learning_rate": 3.243050116103128e-06,
+      "loss": 0.0448,
+      "step": 246
+    },
+    {
+      "epoch": 2.2660550458715596,
+      "grad_norm": 0.07123305648565292,
+      "learning_rate": 3.1679679742830806e-06,
+      "loss": 0.0991,
+      "step": 247
+    },
+    {
+      "epoch": 2.2752293577981653,
+      "grad_norm": 0.05694476515054703,
+      "learning_rate": 3.0936012656763937e-06,
+      "loss": 0.1169,
+      "step": 248
+    },
+    {
+      "epoch": 2.2844036697247705,
+      "grad_norm": 0.04774909466505051,
+      "learning_rate": 3.019957777777788e-06,
+      "loss": 0.0655,
+      "step": 249
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.05831342935562134,
+      "learning_rate": 2.9470452223481206e-06,
+      "loss": 0.0618,
+      "step": 250
+    },
+    {
+      "epoch": 2.302752293577982,
+      "grad_norm": 0.08326123654842377,
+      "learning_rate": 2.8748712346068464e-06,
+      "loss": 0.0923,
+      "step": 251
+    },
+    {
+      "epoch": 2.311926605504587,
+      "grad_norm": 0.06080171838402748,
+      "learning_rate": 2.8034433724324716e-06,
+      "loss": 0.0759,
+      "step": 252
+    },
+    {
+      "epoch": 2.311926605504587,
+      "eval_loss": 0.052994709461927414,
+      "eval_runtime": 43.3989,
+      "eval_samples_per_second": 3.986,
+      "eval_steps_per_second": 2.005,
+      "step": 252
+    },
+    {
+      "epoch": 2.3211009174311927,
+      "grad_norm": 0.06857974827289581,
+      "learning_rate": 2.7327691155710978e-06,
+      "loss": 0.0791,
+      "step": 253
+    },
+    {
+      "epoch": 2.330275229357798,
+      "grad_norm": 0.08638834208250046,
+      "learning_rate": 2.6628558648531845e-06,
+      "loss": 0.0894,
+      "step": 254
+    },
+    {
+      "epoch": 2.3394495412844036,
+      "grad_norm": 0.07400725036859512,
+      "learning_rate": 2.593710941418537e-06,
+      "loss": 0.0624,
+      "step": 255
+    },
+    {
+      "epoch": 2.3486238532110093,
+      "grad_norm": 0.05066663771867752,
+      "learning_rate": 2.525341585949662e-06,
+      "loss": 0.0755,
+      "step": 256
+    },
+    {
+      "epoch": 2.3577981651376145,
+      "grad_norm": 0.051476072520017624,
+      "learning_rate": 2.4577549579135318e-06,
+      "loss": 0.074,
+      "step": 257
+    },
+    {
+      "epoch": 2.36697247706422,
+      "grad_norm": 0.05693186819553375,
+      "learning_rate": 2.3909581348118803e-06,
+      "loss": 0.049,
+      "step": 258
+    },
+    {
+      "epoch": 2.376146788990826,
+      "grad_norm": 0.043929580599069595,
+      "learning_rate": 2.324958111440051e-06,
+      "loss": 0.0558,
+      "step": 259
+    },
+    {
+      "epoch": 2.385321100917431,
+      "grad_norm": 0.06177612394094467,
+      "learning_rate": 2.259761799154516e-06,
+      "loss": 0.0634,
+      "step": 260
+    },
+    {
+      "epoch": 2.3944954128440368,
+      "grad_norm": 0.08901379257440567,
+      "learning_rate": 2.195376025149156e-06,
+      "loss": 0.0552,
+      "step": 261
+    },
+    {
+      "epoch": 2.4036697247706424,
+      "grad_norm": 0.059478871524333954,
+      "learning_rate": 2.1318075317403152e-06,
+      "loss": 0.0834,
+      "step": 262
+    },
+    {
+      "epoch": 2.4128440366972477,
+      "grad_norm": 0.14992526173591614,
+      "learning_rate": 2.069062975660765e-06,
+      "loss": 0.0582,
+      "step": 263
+    },
+    {
+      "epoch": 2.4220183486238533,
+      "grad_norm": 0.04817449301481247,
+      "learning_rate": 2.0071489273626376e-06,
+      "loss": 0.0547,
+      "step": 264
+    },
+    {
+      "epoch": 2.4311926605504586,
+      "grad_norm": 0.08196448534727097,
+      "learning_rate": 1.946071870329377e-06,
+      "loss": 0.078,
+      "step": 265
+    },
+    {
+      "epoch": 2.4403669724770642,
+      "grad_norm": 0.07558903098106384,
+      "learning_rate": 1.885838200396808e-06,
+      "loss": 0.0507,
+      "step": 266
+    },
+    {
+      "epoch": 2.44954128440367,
+      "grad_norm": 0.061492372304201126,
+      "learning_rate": 1.826454225083375e-06,
+      "loss": 0.0526,
+      "step": 267
+    },
+    {
+      "epoch": 2.458715596330275,
+      "grad_norm": 0.04717002436518669,
+      "learning_rate": 1.7679261629296408e-06,
+      "loss": 0.05,
+      "step": 268
+    },
+    {
+      "epoch": 2.467889908256881,
+      "grad_norm": 0.050578705966472626,
+      "learning_rate": 1.7102601428470988e-06,
+      "loss": 0.0694,
+      "step": 269
+    },
+    {
+      "epoch": 2.477064220183486,
+      "grad_norm": 0.06575262546539307,
+      "learning_rate": 1.6534622034763558e-06,
+      "loss": 0.0537,
+      "step": 270
+    },
+    {
+      "epoch": 2.4862385321100917,
+      "grad_norm": 0.0549924410879612,
+      "learning_rate": 1.5975382925547966e-06,
+      "loss": 0.0802,
+      "step": 271
+    },
+    {
+      "epoch": 2.4954128440366974,
+      "grad_norm": 0.06130588427186012,
+      "learning_rate": 1.5424942662937436e-06,
+      "loss": 0.0766,
+      "step": 272
+    },
+    {
+      "epoch": 2.5045871559633026,
+      "grad_norm": 0.07862205803394318,
+      "learning_rate": 1.4883358887652044e-06,
+      "loss": 0.0612,
+      "step": 273
+    },
+    {
+      "epoch": 2.5137614678899083,
+      "grad_norm": 0.04936962202191353,
+      "learning_rate": 1.4350688312982864e-06,
+      "loss": 0.0556,
+      "step": 274
+    },
+    {
+      "epoch": 2.522935779816514,
+      "grad_norm": 0.06410589069128036,
+      "learning_rate": 1.3826986718852952e-06,
+      "loss": 0.0483,
+      "step": 275
+    },
+    {
+      "epoch": 2.532110091743119,
+      "grad_norm": 0.09402082115411758,
+      "learning_rate": 1.3312308945976348e-06,
+      "loss": 0.1031,
+      "step": 276
+    },
+    {
+      "epoch": 2.541284403669725,
+      "grad_norm": 0.052867498248815536,
+      "learning_rate": 1.2806708890115138e-06,
+      "loss": 0.065,
+      "step": 277
+    },
+    {
+      "epoch": 2.5504587155963305,
+      "grad_norm": 0.08837206661701202,
+      "learning_rate": 1.2310239496435749e-06,
+      "loss": 0.095,
+      "step": 278
+    },
+    {
+      "epoch": 2.5596330275229358,
+      "grad_norm": 0.08973362296819687,
+      "learning_rate": 1.1822952753964667e-06,
+      "loss": 0.0765,
+      "step": 279
+    },
+    {
+      "epoch": 2.5688073394495414,
+      "grad_norm": 0.061795495450496674,
+      "learning_rate": 1.134489969014414e-06,
+      "loss": 0.0583,
+      "step": 280
+    },
+    {
+      "epoch": 2.5688073394495414,
+      "eval_loss": 0.05294761061668396,
+      "eval_runtime": 43.223,
+      "eval_samples_per_second": 4.002,
+      "eval_steps_per_second": 2.013,
+      "step": 280
+    },
+    {
+      "epoch": 2.5779816513761467,
+      "grad_norm": 0.047972485423088074,
+      "learning_rate": 1.087613036548888e-06,
+      "loss": 0.048,
+      "step": 281
+    },
+    {
+      "epoch": 2.5871559633027523,
+      "grad_norm": 0.06896362453699112,
+      "learning_rate": 1.0416693868343796e-06,
+      "loss": 0.0771,
+      "step": 282
+    },
+    {
+      "epoch": 2.5963302752293576,
+      "grad_norm": 0.06132780387997627,
+      "learning_rate": 9.966638309743481e-07,
+      "loss": 0.0854,
+      "step": 283
+    },
+    {
+      "epoch": 2.6055045871559632,
+      "grad_norm": 0.06309553980827332,
+      "learning_rate": 9.52601081837431e-07,
+      "loss": 0.0827,
+      "step": 284
+    },
+    {
+      "epoch": 2.614678899082569,
+      "grad_norm": 0.08898341655731201,
+      "learning_rate": 9.094857535639157e-07,
+      "loss": 0.0727,
+      "step": 285
+    },
+    {
+      "epoch": 2.623853211009174,
+      "grad_norm": 0.05615299567580223,
+      "learning_rate": 8.673223610825532e-07,
+      "loss": 0.0827,
+      "step": 286
+    },
+    {
+      "epoch": 2.63302752293578,
+      "grad_norm": 0.06234830617904663,
+      "learning_rate": 8.261153196377814e-07,
+      "loss": 0.0772,
+      "step": 287
+    },
+    {
+      "epoch": 2.6422018348623855,
+      "grad_norm": 0.057416193187236786,
+      "learning_rate": 7.858689443273548e-07,
+      "loss": 0.0726,
+      "step": 288
+    },
+    {
+      "epoch": 2.6513761467889907,
+      "grad_norm": 0.056388452649116516,
+      "learning_rate": 7.465874496504944e-07,
+      "loss": 0.0881,
+      "step": 289
+    },
+    {
+      "epoch": 2.6605504587155964,
+      "grad_norm": 0.05161774531006813,
+      "learning_rate": 7.082749490665353e-07,
+      "loss": 0.0447,
+      "step": 290
+    },
+    {
+      "epoch": 2.669724770642202,
+      "grad_norm": 0.047958966344594955,
+      "learning_rate": 6.709354545641989e-07,
+      "loss": 0.0772,
+      "step": 291
+    },
+    {
+      "epoch": 2.6788990825688073,
+      "grad_norm": 0.0640062615275383,
+      "learning_rate": 6.345728762414504e-07,
+      "loss": 0.0607,
+      "step": 292
+    },
+    {
+      "epoch": 2.688073394495413,
+      "grad_norm": 0.05299694091081619,
+      "learning_rate": 5.99191021896055e-07,
+      "loss": 0.047,
+      "step": 293
+    },
+    {
+      "epoch": 2.6972477064220186,
+      "grad_norm": 0.057945024222135544,
+      "learning_rate": 5.647935966268225e-07,
+      "loss": 0.0731,
+      "step": 294
+    },
+    {
+      "epoch": 2.706422018348624,
+      "grad_norm": 0.05141222104430199,
+      "learning_rate": 5.313842024456306e-07,
+      "loss": 0.039,
+      "step": 295
+    },
+    {
+      "epoch": 2.7155963302752295,
+      "grad_norm": 0.08650866150856018,
+      "learning_rate": 4.98966337900224e-07,
+      "loss": 0.0551,
+      "step": 296
+    },
+    {
+      "epoch": 2.7247706422018347,
+      "grad_norm": 0.13347071409225464,
+      "learning_rate": 4.6754339770785474e-07,
+      "loss": 0.0619,
+      "step": 297
+    },
+    {
+      "epoch": 2.7339449541284404,
+      "grad_norm": 0.07346609234809875,
+      "learning_rate": 4.3711867239980335e-07,
+      "loss": 0.0423,
+      "step": 298
+    },
+    {
+      "epoch": 2.7431192660550456,
+      "grad_norm": 0.05696272850036621,
+      "learning_rate": 4.076953479767964e-07,
+      "loss": 0.0903,
+      "step": 299
+    },
+    {
+      "epoch": 2.7522935779816513,
+      "grad_norm": 0.058090586215257645,
+      "learning_rate": 3.792765055753755e-07,
+      "loss": 0.0994,
+      "step": 300
+    },
+    {
+      "epoch": 2.761467889908257,
+      "grad_norm": 0.06176576018333435,
+      "learning_rate": 3.5186512114525283e-07,
+      "loss": 0.1243,
+      "step": 301
+    },
+    {
+      "epoch": 2.770642201834862,
+      "grad_norm": 0.07493139058351517,
+      "learning_rate": 3.25464065137675e-07,
+      "loss": 0.0584,
+      "step": 302
+    },
+    {
+      "epoch": 2.779816513761468,
+      "grad_norm": 0.04847017675638199,
+      "learning_rate": 3.0007610220483927e-07,
+      "loss": 0.0652,
+      "step": 303
+    },
+    {
+      "epoch": 2.7889908256880735,
+      "grad_norm": 0.058301348239183426,
+      "learning_rate": 2.757038909103793e-07,
+      "loss": 0.0708,
+      "step": 304
+    },
+    {
+      "epoch": 2.7981651376146788,
+      "grad_norm": 0.046219125390052795,
+      "learning_rate": 2.523499834509724e-07,
+      "loss": 0.0495,
+      "step": 305
+    },
+    {
+      "epoch": 2.8073394495412844,
+      "grad_norm": 0.051395233720541,
+      "learning_rate": 2.3001682538908333e-07,
+      "loss": 0.0954,
+      "step": 306
+    },
+    {
+      "epoch": 2.81651376146789,
+      "grad_norm": 0.0680239349603653,
+      "learning_rate": 2.0870675539686024e-07,
+      "loss": 0.0717,
+      "step": 307
+    },
+    {
+      "epoch": 2.8256880733944953,
+      "grad_norm": 0.058481365442276,
+      "learning_rate": 1.884220050112462e-07,
+      "loss": 0.1087,
+      "step": 308
+    },
+    {
+      "epoch": 2.8256880733944953,
+      "eval_loss": 0.05293623358011246,
+      "eval_runtime": 43.297,
+      "eval_samples_per_second": 3.996,
+      "eval_steps_per_second": 2.009,
+      "step": 308
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 327,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 28,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.9114921838279393e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-308/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8691334e7ff9d485bc39601a29a4096723c23e5fb7323cdb19a40a1c9c993c02
+size 6520

checkpoint-327/README.md

@@ -0,0 +1,202 @@
+---
+base_model: bigcode/starcoder2-15b
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

checkpoint-327/adapter_config.json

@@ -0,0 +1,36 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "bigcode/starcoder2-15b",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_fc",
+    "k_proj",
+    "q_proj",
+    "o_proj",
+    "c_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-327/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c90ba47f39c0a86cd549d28055b620e1922d97f151fa40bf696f678998cec60
+size 508623712

checkpoint-327/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f42239acaef6ca722dabee11b35f50eb2b911f45924c3928e00f1ddb0b34b5f
+size 258817236

checkpoint-327/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03bce6c25ff9b0b569496e5db6947cc523f92f36cbda69995fac679c1bb7c540
+size 14244

checkpoint-327/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fedb7ca125efe47bff8669910b66d94c1ed551f2a3528c1e3d15fd2eeb7a204
+size 1064

checkpoint-327/special_tokens_map.json

@@ -0,0 +1,70 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-327/tokenizer_config.json

@@ -0,0 +1,358 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "37": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

checkpoint-327/trainer_state.json

@@ -0,0 +1,2418 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 28,
+  "global_step": 327,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009174311926605505,
+      "grad_norm": 0.01852019689977169,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1499,
+      "step": 1
+    },
+    {
+      "epoch": 0.009174311926605505,
+      "eval_loss": 0.06453218311071396,
+      "eval_runtime": 43.1581,
+      "eval_samples_per_second": 4.009,
+      "eval_steps_per_second": 2.016,
+      "step": 1
+    },
+    {
+      "epoch": 0.01834862385321101,
+      "grad_norm": 0.02213262766599655,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.1596,
+      "step": 2
+    },
+    {
+      "epoch": 0.027522935779816515,
+      "grad_norm": 0.045894358307123184,
+      "learning_rate": 3e-06,
+      "loss": 0.1794,
+      "step": 3
+    },
+    {
+      "epoch": 0.03669724770642202,
+      "grad_norm": 0.01868272013962269,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.1592,
+      "step": 4
+    },
+    {
+      "epoch": 0.045871559633027525,
+      "grad_norm": 0.017406364902853966,
+      "learning_rate": 5e-06,
+      "loss": 0.1696,
+      "step": 5
+    },
+    {
+      "epoch": 0.05504587155963303,
+      "grad_norm": 0.01861901767551899,
+      "learning_rate": 6e-06,
+      "loss": 0.1438,
+      "step": 6
+    },
+    {
+      "epoch": 0.06422018348623854,
+      "grad_norm": 0.020326999947428703,
+      "learning_rate": 7e-06,
+      "loss": 0.1627,
+      "step": 7
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 0.02542084828019142,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.1558,
+      "step": 8
+    },
+    {
+      "epoch": 0.08256880733944955,
+      "grad_norm": 0.022425655275583267,
+      "learning_rate": 9e-06,
+      "loss": 0.1517,
+      "step": 9
+    },
+    {
+      "epoch": 0.09174311926605505,
+      "grad_norm": 0.023916274309158325,
+      "learning_rate": 1e-05,
+      "loss": 0.1672,
+      "step": 10
+    },
+    {
+      "epoch": 0.10091743119266056,
+      "grad_norm": 0.022392459213733673,
+      "learning_rate": 1.1000000000000001e-05,
+      "loss": 0.1504,
+      "step": 11
+    },
+    {
+      "epoch": 0.11009174311926606,
+      "grad_norm": 0.02520025707781315,
+      "learning_rate": 1.2e-05,
+      "loss": 0.1749,
+      "step": 12
+    },
+    {
+      "epoch": 0.11926605504587157,
+      "grad_norm": 0.028882062062621117,
+      "learning_rate": 1.3000000000000001e-05,
+      "loss": 0.1705,
+      "step": 13
+    },
+    {
+      "epoch": 0.12844036697247707,
+      "grad_norm": 0.03628265857696533,
+      "learning_rate": 1.4e-05,
+      "loss": 0.166,
+      "step": 14
+    },
+    {
+      "epoch": 0.13761467889908258,
+      "grad_norm": 0.02980518713593483,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 0.1238,
+      "step": 15
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 0.028387300670146942,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.1326,
+      "step": 16
+    },
+    {
+      "epoch": 0.1559633027522936,
+      "grad_norm": 0.03367001935839653,
+      "learning_rate": 1.7e-05,
+      "loss": 0.1347,
+      "step": 17
+    },
+    {
+      "epoch": 0.1651376146788991,
+      "grad_norm": 0.03655758127570152,
+      "learning_rate": 1.8e-05,
+      "loss": 0.1423,
+      "step": 18
+    },
+    {
+      "epoch": 0.1743119266055046,
+      "grad_norm": 0.04000673070549965,
+      "learning_rate": 1.9e-05,
+      "loss": 0.1651,
+      "step": 19
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 0.03844478353857994,
+      "learning_rate": 2e-05,
+      "loss": 0.1649,
+      "step": 20
+    },
+    {
+      "epoch": 0.1926605504587156,
+      "grad_norm": 0.04334944114089012,
+      "learning_rate": 1.99994764125734e-05,
+      "loss": 0.1292,
+      "step": 21
+    },
+    {
+      "epoch": 0.2018348623853211,
+      "grad_norm": 0.04224175587296486,
+      "learning_rate": 1.9997905705122352e-05,
+      "loss": 0.1336,
+      "step": 22
+    },
+    {
+      "epoch": 0.21100917431192662,
+      "grad_norm": 0.04647228121757507,
+      "learning_rate": 1.9995288042127396e-05,
+      "loss": 0.128,
+      "step": 23
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 0.04862922057509422,
+      "learning_rate": 1.9991623697703613e-05,
+      "loss": 0.1611,
+      "step": 24
+    },
+    {
+      "epoch": 0.22935779816513763,
+      "grad_norm": 0.05552718788385391,
+      "learning_rate": 1.998691305557194e-05,
+      "loss": 0.1469,
+      "step": 25
+    },
+    {
+      "epoch": 0.23853211009174313,
+      "grad_norm": 0.04372956603765488,
+      "learning_rate": 1.9981156609018977e-05,
+      "loss": 0.1341,
+      "step": 26
+    },
+    {
+      "epoch": 0.24770642201834864,
+      "grad_norm": 0.04992978647351265,
+      "learning_rate": 1.9974354960845326e-05,
+      "loss": 0.1464,
+      "step": 27
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "grad_norm": 0.04873019829392433,
+      "learning_rate": 1.9966508823302484e-05,
+      "loss": 0.1554,
+      "step": 28
+    },
+    {
+      "epoch": 0.25688073394495414,
+      "eval_loss": 0.06217445060610771,
+      "eval_runtime": 43.3972,
+      "eval_samples_per_second": 3.986,
+      "eval_steps_per_second": 2.005,
+      "step": 28
+    },
+    {
+      "epoch": 0.26605504587155965,
+      "grad_norm": 0.04296933487057686,
+      "learning_rate": 1.9957619018018243e-05,
+      "loss": 0.1231,
+      "step": 29
+    },
+    {
+      "epoch": 0.27522935779816515,
+      "grad_norm": 0.06265883892774582,
+      "learning_rate": 1.9947686475910656e-05,
+      "loss": 0.1292,
+      "step": 30
+    },
+    {
+      "epoch": 0.28440366972477066,
+      "grad_norm": 0.044797539710998535,
+      "learning_rate": 1.9936712237090554e-05,
+      "loss": 0.114,
+      "step": 31
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 0.05862847715616226,
+      "learning_rate": 1.9924697450752636e-05,
+      "loss": 0.1215,
+      "step": 32
+    },
+    {
+      "epoch": 0.30275229357798167,
+      "grad_norm": 0.04938759654760361,
+      "learning_rate": 1.991164337505511e-05,
+      "loss": 0.0683,
+      "step": 33
+    },
+    {
+      "epoch": 0.3119266055045872,
+      "grad_norm": 0.07766549289226532,
+      "learning_rate": 1.9897551376987948e-05,
+      "loss": 0.1051,
+      "step": 34
+    },
+    {
+      "epoch": 0.3211009174311927,
+      "grad_norm": 0.05474488437175751,
+      "learning_rate": 1.9882422932229765e-05,
+      "loss": 0.082,
+      "step": 35
+    },
+    {
+      "epoch": 0.3302752293577982,
+      "grad_norm": 0.04499150812625885,
+      "learning_rate": 1.9866259624993246e-05,
+      "loss": 0.1135,
+      "step": 36
+    },
+    {
+      "epoch": 0.3394495412844037,
+      "grad_norm": 0.07329924404621124,
+      "learning_rate": 1.9849063147859282e-05,
+      "loss": 0.1082,
+      "step": 37
+    },
+    {
+      "epoch": 0.3486238532110092,
+      "grad_norm": 0.23288355767726898,
+      "learning_rate": 1.983083530159971e-05,
+      "loss": 0.0899,
+      "step": 38
+    },
+    {
+      "epoch": 0.3577981651376147,
+      "grad_norm": 0.06561094522476196,
+      "learning_rate": 1.9811577994988755e-05,
+      "loss": 0.1096,
+      "step": 39
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.052528683096170425,
+      "learning_rate": 1.979129324460314e-05,
+      "loss": 0.107,
+      "step": 40
+    },
+    {
+      "epoch": 0.3761467889908257,
+      "grad_norm": 0.057943329215049744,
+      "learning_rate": 1.9769983174610918e-05,
+      "loss": 0.1121,
+      "step": 41
+    },
+    {
+      "epoch": 0.3853211009174312,
+      "grad_norm": 0.05784667655825615,
+      "learning_rate": 1.974765001654903e-05,
+      "loss": 0.1125,
+      "step": 42
+    },
+    {
+      "epoch": 0.3944954128440367,
+      "grad_norm": 0.04998760297894478,
+      "learning_rate": 1.9724296109089623e-05,
+      "loss": 0.0944,
+      "step": 43
+    },
+    {
+      "epoch": 0.4036697247706422,
+      "grad_norm": 0.056932978332042694,
+      "learning_rate": 1.9699923897795165e-05,
+      "loss": 0.0758,
+      "step": 44
+    },
+    {
+      "epoch": 0.41284403669724773,
+      "grad_norm": 0.05268337205052376,
+      "learning_rate": 1.9674535934862327e-05,
+      "loss": 0.0767,
+      "step": 45
+    },
+    {
+      "epoch": 0.42201834862385323,
+      "grad_norm": 0.04703257977962494,
+      "learning_rate": 1.9648134878854747e-05,
+      "loss": 0.076,
+      "step": 46
+    },
+    {
+      "epoch": 0.43119266055045874,
+      "grad_norm": 0.05632725730538368,
+      "learning_rate": 1.9620723494424627e-05,
+      "loss": 0.1143,
+      "step": 47
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 0.04887419193983078,
+      "learning_rate": 1.9592304652023208e-05,
+      "loss": 0.096,
+      "step": 48
+    },
+    {
+      "epoch": 0.44954128440366975,
+      "grad_norm": 0.06641880422830582,
+      "learning_rate": 1.9562881327600197e-05,
+      "loss": 0.1108,
+      "step": 49
+    },
+    {
+      "epoch": 0.45871559633027525,
+      "grad_norm": 0.08709923177957535,
+      "learning_rate": 1.9532456602292148e-05,
+      "loss": 0.0987,
+      "step": 50
+    },
+    {
+      "epoch": 0.46788990825688076,
+      "grad_norm": 0.06175887584686279,
+      "learning_rate": 1.950103366209978e-05,
+      "loss": 0.0821,
+      "step": 51
+    },
+    {
+      "epoch": 0.47706422018348627,
+      "grad_norm": 0.05565601587295532,
+      "learning_rate": 1.9468615797554374e-05,
+      "loss": 0.0727,
+      "step": 52
+    },
+    {
+      "epoch": 0.48623853211009177,
+      "grad_norm": 0.13676409423351288,
+      "learning_rate": 1.943520640337318e-05,
+      "loss": 0.0834,
+      "step": 53
+    },
+    {
+      "epoch": 0.4954128440366973,
+      "grad_norm": 0.0817922055721283,
+      "learning_rate": 1.9400808978103948e-05,
+      "loss": 0.0766,
+      "step": 54
+    },
+    {
+      "epoch": 0.5045871559633027,
+      "grad_norm": 0.04707655310630798,
+      "learning_rate": 1.936542712375855e-05,
+      "loss": 0.0753,
+      "step": 55
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 0.07192892581224442,
+      "learning_rate": 1.9329064545435803e-05,
+      "loss": 0.0745,
+      "step": 56
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "eval_loss": 0.057128190994262695,
+      "eval_runtime": 43.3864,
+      "eval_samples_per_second": 3.987,
+      "eval_steps_per_second": 2.005,
+      "step": 56
+    },
+    {
+      "epoch": 0.5229357798165137,
+      "grad_norm": 0.07009316980838776,
+      "learning_rate": 1.929172505093347e-05,
+      "loss": 0.0696,
+      "step": 57
+    },
+    {
+      "epoch": 0.5321100917431193,
+      "grad_norm": 0.07050078362226486,
+      "learning_rate": 1.9253412550349507e-05,
+      "loss": 0.0846,
+      "step": 58
+    },
+    {
+      "epoch": 0.5412844036697247,
+      "grad_norm": 0.06861168146133423,
+      "learning_rate": 1.9214131055672648e-05,
+      "loss": 0.0659,
+      "step": 59
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.05825705826282501,
+      "learning_rate": 1.917388468036222e-05,
+      "loss": 0.1173,
+      "step": 60
+    },
+    {
+      "epoch": 0.5596330275229358,
+      "grad_norm": 0.11085808277130127,
+      "learning_rate": 1.913267763891745e-05,
+      "loss": 0.0715,
+      "step": 61
+    },
+    {
+      "epoch": 0.5688073394495413,
+      "grad_norm": 0.0637730062007904,
+      "learning_rate": 1.9090514246436085e-05,
+      "loss": 0.0936,
+      "step": 62
+    },
+    {
+      "epoch": 0.5779816513761468,
+      "grad_norm": 0.05786406993865967,
+      "learning_rate": 1.904739891816257e-05,
+      "loss": 0.0777,
+      "step": 63
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 0.09354288130998611,
+      "learning_rate": 1.9003336169025655e-05,
+      "loss": 0.0913,
+      "step": 64
+    },
+    {
+      "epoch": 0.5963302752293578,
+      "grad_norm": 0.1899336725473404,
+      "learning_rate": 1.8958330613165622e-05,
+      "loss": 0.0761,
+      "step": 65
+    },
+    {
+      "epoch": 0.6055045871559633,
+      "grad_norm": 0.05038751661777496,
+      "learning_rate": 1.891238696345111e-05,
+      "loss": 0.078,
+      "step": 66
+    },
+    {
+      "epoch": 0.6146788990825688,
+      "grad_norm": 0.16387197375297546,
+      "learning_rate": 1.8865510030985588e-05,
+      "loss": 0.0917,
+      "step": 67
+    },
+    {
+      "epoch": 0.6238532110091743,
+      "grad_norm": 0.054418135434389114,
+      "learning_rate": 1.8817704724603536e-05,
+      "loss": 0.0792,
+      "step": 68
+    },
+    {
+      "epoch": 0.6330275229357798,
+      "grad_norm": 0.0637287050485611,
+      "learning_rate": 1.8768976050356428e-05,
+      "loss": 0.0731,
+      "step": 69
+    },
+    {
+      "epoch": 0.6422018348623854,
+      "grad_norm": 0.27382200956344604,
+      "learning_rate": 1.8719329110988487e-05,
+      "loss": 0.0745,
+      "step": 70
+    },
+    {
+      "epoch": 0.6513761467889908,
+      "grad_norm": 0.05016474053263664,
+      "learning_rate": 1.8668769105402366e-05,
+      "loss": 0.0739,
+      "step": 71
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 0.08013670146465302,
+      "learning_rate": 1.8617301328114704e-05,
+      "loss": 0.1024,
+      "step": 72
+    },
+    {
+      "epoch": 0.6697247706422018,
+      "grad_norm": 0.06279715895652771,
+      "learning_rate": 1.8564931168701713e-05,
+      "loss": 0.0971,
+      "step": 73
+    },
+    {
+      "epoch": 0.6788990825688074,
+      "grad_norm": 0.048747751861810684,
+      "learning_rate": 1.85116641112348e-05,
+      "loss": 0.0657,
+      "step": 74
+    },
+    {
+      "epoch": 0.6880733944954128,
+      "grad_norm": 0.05987577140331268,
+      "learning_rate": 1.845750573370626e-05,
+      "loss": 0.1191,
+      "step": 75
+    },
+    {
+      "epoch": 0.6972477064220184,
+      "grad_norm": 0.08020060509443283,
+      "learning_rate": 1.8402461707445206e-05,
+      "loss": 0.1112,
+      "step": 76
+    },
+    {
+      "epoch": 0.7064220183486238,
+      "grad_norm": 0.05764961615204811,
+      "learning_rate": 1.8346537796523643e-05,
+      "loss": 0.1049,
+      "step": 77
+    },
+    {
+      "epoch": 0.7155963302752294,
+      "grad_norm": 0.08143055438995361,
+      "learning_rate": 1.8289739857152903e-05,
+      "loss": 0.0871,
+      "step": 78
+    },
+    {
+      "epoch": 0.7247706422018348,
+      "grad_norm": 0.08280878514051437,
+      "learning_rate": 1.823207383707036e-05,
+      "loss": 0.0724,
+      "step": 79
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.08486371487379074,
+      "learning_rate": 1.8173545774916628e-05,
+      "loss": 0.068,
+      "step": 80
+    },
+    {
+      "epoch": 0.7431192660550459,
+      "grad_norm": 0.061856675893068314,
+      "learning_rate": 1.8114161799603195e-05,
+      "loss": 0.0786,
+      "step": 81
+    },
+    {
+      "epoch": 0.7522935779816514,
+      "grad_norm": 0.05205192044377327,
+      "learning_rate": 1.8053928129670624e-05,
+      "loss": 0.0755,
+      "step": 82
+    },
+    {
+      "epoch": 0.7614678899082569,
+      "grad_norm": 0.07000340521335602,
+      "learning_rate": 1.7992851072637366e-05,
+      "loss": 0.1239,
+      "step": 83
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "grad_norm": 0.07025006413459778,
+      "learning_rate": 1.793093702433924e-05,
+      "loss": 0.0616,
+      "step": 84
+    },
+    {
+      "epoch": 0.7706422018348624,
+      "eval_loss": 0.055874165147542953,
+      "eval_runtime": 43.2177,
+      "eval_samples_per_second": 4.003,
+      "eval_steps_per_second": 2.013,
+      "step": 84
+    },
+    {
+      "epoch": 0.7798165137614679,
+      "grad_norm": 0.06147678196430206,
+      "learning_rate": 1.7868192468259686e-05,
+      "loss": 0.0917,
+      "step": 85
+    },
+    {
+      "epoch": 0.7889908256880734,
+      "grad_norm": 0.045870471745729446,
+      "learning_rate": 1.7804623974850844e-05,
+      "loss": 0.092,
+      "step": 86
+    },
+    {
+      "epoch": 0.7981651376146789,
+      "grad_norm": 0.07622863352298737,
+      "learning_rate": 1.7740238200845485e-05,
+      "loss": 0.0983,
+      "step": 87
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 0.08215321600437164,
+      "learning_rate": 1.7675041888559952e-05,
+      "loss": 0.0971,
+      "step": 88
+    },
+    {
+      "epoch": 0.8165137614678899,
+      "grad_norm": 0.06286073476076126,
+      "learning_rate": 1.7609041865188122e-05,
+      "loss": 0.0876,
+      "step": 89
+    },
+    {
+      "epoch": 0.8256880733944955,
+      "grad_norm": 0.06399139016866684,
+      "learning_rate": 1.754224504208647e-05,
+      "loss": 0.1166,
+      "step": 90
+    },
+    {
+      "epoch": 0.8348623853211009,
+      "grad_norm": 0.07115256041288376,
+      "learning_rate": 1.7474658414050344e-05,
+      "loss": 0.119,
+      "step": 91
+    },
+    {
+      "epoch": 0.8440366972477065,
+      "grad_norm": 0.04867429658770561,
+      "learning_rate": 1.7406289058581466e-05,
+      "loss": 0.0609,
+      "step": 92
+    },
+    {
+      "epoch": 0.8532110091743119,
+      "grad_norm": 0.1833045333623886,
+      "learning_rate": 1.7337144135146818e-05,
+      "loss": 0.1247,
+      "step": 93
+    },
+    {
+      "epoch": 0.8623853211009175,
+      "grad_norm": 0.07023169845342636,
+      "learning_rate": 1.7267230884428905e-05,
+      "loss": 0.1271,
+      "step": 94
+    },
+    {
+      "epoch": 0.8715596330275229,
+      "grad_norm": 0.044201672077178955,
+      "learning_rate": 1.719655662756753e-05,
+      "loss": 0.0661,
+      "step": 95
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 0.05139552056789398,
+      "learning_rate": 1.7125128765393157e-05,
+      "loss": 0.1065,
+      "step": 96
+    },
+    {
+      "epoch": 0.8899082568807339,
+      "grad_norm": 0.06401531398296356,
+      "learning_rate": 1.705295477765188e-05,
+      "loss": 0.1006,
+      "step": 97
+    },
+    {
+      "epoch": 0.8990825688073395,
+      "grad_norm": 0.04898101091384888,
+      "learning_rate": 1.6980042222222216e-05,
+      "loss": 0.0522,
+      "step": 98
+    },
+    {
+      "epoch": 0.908256880733945,
+      "grad_norm": 0.053434859961271286,
+      "learning_rate": 1.690639873432361e-05,
+      "loss": 0.1214,
+      "step": 99
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.058289702981710434,
+      "learning_rate": 1.683203202571692e-05,
+      "loss": 0.0547,
+      "step": 100
+    },
+    {
+      "epoch": 0.926605504587156,
+      "grad_norm": 0.06972479820251465,
+      "learning_rate": 1.6756949883896874e-05,
+      "loss": 0.0867,
+      "step": 101
+    },
+    {
+      "epoch": 0.9357798165137615,
+      "grad_norm": 0.05255963280797005,
+      "learning_rate": 1.668116017127655e-05,
+      "loss": 0.0685,
+      "step": 102
+    },
+    {
+      "epoch": 0.944954128440367,
+      "grad_norm": 0.04849875345826149,
+      "learning_rate": 1.6604670824364067e-05,
+      "loss": 0.074,
+      "step": 103
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 0.08504751324653625,
+      "learning_rate": 1.652748985293149e-05,
+      "loss": 0.0901,
+      "step": 104
+    },
+    {
+      "epoch": 0.963302752293578,
+      "grad_norm": 0.07853557169437408,
+      "learning_rate": 1.6449625339176056e-05,
+      "loss": 0.0663,
+      "step": 105
+    },
+    {
+      "epoch": 0.9724770642201835,
+      "grad_norm": 0.0786280408501625,
+      "learning_rate": 1.6371085436873847e-05,
+      "loss": 0.0936,
+      "step": 106
+    },
+    {
+      "epoch": 0.981651376146789,
+      "grad_norm": 0.09738124907016754,
+      "learning_rate": 1.6291878370525925e-05,
+      "loss": 0.1001,
+      "step": 107
+    },
+    {
+      "epoch": 0.9908256880733946,
+      "grad_norm": 0.061633653938770294,
+      "learning_rate": 1.6212012434497103e-05,
+      "loss": 0.0728,
+      "step": 108
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.06726932525634766,
+      "learning_rate": 1.6131495992147363e-05,
+      "loss": 0.0837,
+      "step": 109
+    },
+    {
+      "epoch": 1.0091743119266054,
+      "grad_norm": 0.05419926717877388,
+      "learning_rate": 1.605033747495607e-05,
+      "loss": 0.0841,
+      "step": 110
+    },
+    {
+      "epoch": 1.018348623853211,
+      "grad_norm": 0.056966476142406464,
+      "learning_rate": 1.596854538163906e-05,
+      "loss": 0.0774,
+      "step": 111
+    },
+    {
+      "epoch": 1.0275229357798166,
+      "grad_norm": 0.05710803344845772,
+      "learning_rate": 1.5886128277258665e-05,
+      "loss": 0.0645,
+      "step": 112
+    },
+    {
+      "epoch": 1.0275229357798166,
+      "eval_loss": 0.05474008619785309,
+      "eval_runtime": 43.3773,
+      "eval_samples_per_second": 3.988,
+      "eval_steps_per_second": 2.006,
+      "step": 112
+    },
+    {
+      "epoch": 1.036697247706422,
+      "grad_norm": 0.05264132842421532,
+      "learning_rate": 1.58030947923268e-05,
+      "loss": 0.0615,
+      "step": 113
+    },
+    {
+      "epoch": 1.0458715596330275,
+      "grad_norm": 0.11162517964839935,
+      "learning_rate": 1.571945362190121e-05,
+      "loss": 0.13,
+      "step": 114
+    },
+    {
+      "epoch": 1.0550458715596331,
+      "grad_norm": 0.05422775819897652,
+      "learning_rate": 1.563521352467493e-05,
+      "loss": 0.0677,
+      "step": 115
+    },
+    {
+      "epoch": 1.0642201834862386,
+      "grad_norm": 0.08082108199596405,
+      "learning_rate": 1.55503833220591e-05,
+      "loss": 0.0958,
+      "step": 116
+    },
+    {
+      "epoch": 1.073394495412844,
+      "grad_norm": 0.0667729526758194,
+      "learning_rate": 1.546497189725922e-05,
+      "loss": 0.1132,
+      "step": 117
+    },
+    {
+      "epoch": 1.0825688073394495,
+      "grad_norm": 0.06297166645526886,
+      "learning_rate": 1.5378988194344913e-05,
+      "loss": 0.0646,
+      "step": 118
+    },
+    {
+      "epoch": 1.091743119266055,
+      "grad_norm": 0.05654435604810715,
+      "learning_rate": 1.5292441217313324e-05,
+      "loss": 0.1083,
+      "step": 119
+    },
+    {
+      "epoch": 1.1009174311926606,
+      "grad_norm": 0.054773300886154175,
+      "learning_rate": 1.5205340029146256e-05,
+      "loss": 0.0833,
+      "step": 120
+    },
+    {
+      "epoch": 1.110091743119266,
+      "grad_norm": 0.0510234571993351,
+      "learning_rate": 1.5117693750861096e-05,
+      "loss": 0.0792,
+      "step": 121
+    },
+    {
+      "epoch": 1.1192660550458715,
+      "grad_norm": 0.054006725549697876,
+      "learning_rate": 1.5029511560555707e-05,
+      "loss": 0.1066,
+      "step": 122
+    },
+    {
+      "epoch": 1.1284403669724772,
+      "grad_norm": 0.06665553152561188,
+      "learning_rate": 1.4940802692447306e-05,
+      "loss": 0.0584,
+      "step": 123
+    },
+    {
+      "epoch": 1.1376146788990826,
+      "grad_norm": 0.105413518846035,
+      "learning_rate": 1.4851576435905489e-05,
+      "loss": 0.0907,
+      "step": 124
+    },
+    {
+      "epoch": 1.146788990825688,
+      "grad_norm": 0.0626402273774147,
+      "learning_rate": 1.4761842134479463e-05,
+      "loss": 0.0544,
+      "step": 125
+    },
+    {
+      "epoch": 1.1559633027522935,
+      "grad_norm": 0.06280255317687988,
+      "learning_rate": 1.4671609184919622e-05,
+      "loss": 0.0487,
+      "step": 126
+    },
+    {
+      "epoch": 1.165137614678899,
+      "grad_norm": 0.07118494808673859,
+      "learning_rate": 1.4580887036193539e-05,
+      "loss": 0.0797,
+      "step": 127
+    },
+    {
+      "epoch": 1.1743119266055047,
+      "grad_norm": 0.0436442606151104,
+      "learning_rate": 1.4489685188496488e-05,
+      "loss": 0.0568,
+      "step": 128
+    },
+    {
+      "epoch": 1.18348623853211,
+      "grad_norm": 0.06852032989263535,
+      "learning_rate": 1.4398013192256615e-05,
+      "loss": 0.1099,
+      "step": 129
+    },
+    {
+      "epoch": 1.1926605504587156,
+      "grad_norm": 0.05673675611615181,
+      "learning_rate": 1.4305880647134847e-05,
+      "loss": 0.0649,
+      "step": 130
+    },
+    {
+      "epoch": 1.2018348623853212,
+      "grad_norm": 0.05407770350575447,
+      "learning_rate": 1.4213297201019618e-05,
+      "loss": 0.0582,
+      "step": 131
+    },
+    {
+      "epoch": 1.2110091743119267,
+      "grad_norm": 0.06976728141307831,
+      "learning_rate": 1.4120272549016591e-05,
+      "loss": 0.0744,
+      "step": 132
+    },
+    {
+      "epoch": 1.2201834862385321,
+      "grad_norm": 0.044198598712682724,
+      "learning_rate": 1.40268164324334e-05,
+      "loss": 0.0456,
+      "step": 133
+    },
+    {
+      "epoch": 1.2293577981651376,
+      "grad_norm": 0.052981797605752945,
+      "learning_rate": 1.3932938637759555e-05,
+      "loss": 0.067,
+      "step": 134
+    },
+    {
+      "epoch": 1.238532110091743,
+      "grad_norm": 0.051931336522102356,
+      "learning_rate": 1.3838648995641645e-05,
+      "loss": 0.0662,
+      "step": 135
+    },
+    {
+      "epoch": 1.2477064220183487,
+      "grad_norm": 0.06196126341819763,
+      "learning_rate": 1.3743957379853885e-05,
+      "loss": 0.0742,
+      "step": 136
+    },
+    {
+      "epoch": 1.2568807339449541,
+      "grad_norm": 0.0772649347782135,
+      "learning_rate": 1.3648873706264159e-05,
+      "loss": 0.1016,
+      "step": 137
+    },
+    {
+      "epoch": 1.2660550458715596,
+      "grad_norm": 0.06751381605863571,
+      "learning_rate": 1.3553407931795662e-05,
+      "loss": 0.0871,
+      "step": 138
+    },
+    {
+      "epoch": 1.2752293577981653,
+      "grad_norm": 0.045472558587789536,
+      "learning_rate": 1.3457570053384225e-05,
+      "loss": 0.0548,
+      "step": 139
+    },
+    {
+      "epoch": 1.2844036697247707,
+      "grad_norm": 0.061834368854761124,
+      "learning_rate": 1.3361370106931486e-05,
+      "loss": 0.0601,
+      "step": 140
+    },
+    {
+      "epoch": 1.2844036697247707,
+      "eval_loss": 0.05419979989528656,
+      "eval_runtime": 43.2558,
+      "eval_samples_per_second": 3.999,
+      "eval_steps_per_second": 2.011,
+      "step": 140
+    },
+    {
+      "epoch": 1.2935779816513762,
+      "grad_norm": 0.062169574201107025,
+      "learning_rate": 1.3264818166253917e-05,
+      "loss": 0.0899,
+      "step": 141
+    },
+    {
+      "epoch": 1.3027522935779816,
+      "grad_norm": 0.051288578659296036,
+      "learning_rate": 1.3167924342027947e-05,
+      "loss": 0.0758,
+      "step": 142
+    },
+    {
+      "epoch": 1.311926605504587,
+      "grad_norm": 0.052053675055503845,
+      "learning_rate": 1.3070698780731194e-05,
+      "loss": 0.0989,
+      "step": 143
+    },
+    {
+      "epoch": 1.3211009174311927,
+      "grad_norm": 0.05368887633085251,
+      "learning_rate": 1.2973151663579948e-05,
+      "loss": 0.0599,
+      "step": 144
+    },
+    {
+      "epoch": 1.3302752293577982,
+      "grad_norm": 0.06439553946256638,
+      "learning_rate": 1.2875293205463018e-05,
+      "loss": 0.0808,
+      "step": 145
+    },
+    {
+      "epoch": 1.3394495412844036,
+      "grad_norm": 0.05169299989938736,
+      "learning_rate": 1.277713365387205e-05,
+      "loss": 0.0668,
+      "step": 146
+    },
+    {
+      "epoch": 1.3486238532110093,
+      "grad_norm": 0.0563080795109272,
+      "learning_rate": 1.2678683287828451e-05,
+      "loss": 0.0888,
+      "step": 147
+    },
+    {
+      "epoch": 1.3577981651376148,
+      "grad_norm": 0.06615187972784042,
+      "learning_rate": 1.257995241680698e-05,
+      "loss": 0.123,
+      "step": 148
+    },
+    {
+      "epoch": 1.3669724770642202,
+      "grad_norm": 0.0562995970249176,
+      "learning_rate": 1.2480951379656175e-05,
+      "loss": 0.0709,
+      "step": 149
+    },
+    {
+      "epoch": 1.3761467889908257,
+      "grad_norm": 0.06163397431373596,
+      "learning_rate": 1.2381690543515692e-05,
+      "loss": 0.0663,
+      "step": 150
+    },
+    {
+      "epoch": 1.385321100917431,
+      "grad_norm": 0.053988683968782425,
+      "learning_rate": 1.2282180302730683e-05,
+      "loss": 0.0812,
+      "step": 151
+    },
+    {
+      "epoch": 1.3944954128440368,
+      "grad_norm": 0.062011417001485825,
+      "learning_rate": 1.2182431077763317e-05,
+      "loss": 0.0821,
+      "step": 152
+    },
+    {
+      "epoch": 1.4036697247706422,
+      "grad_norm": 0.054933421313762665,
+      "learning_rate": 1.2082453314101607e-05,
+      "loss": 0.0726,
+      "step": 153
+    },
+    {
+      "epoch": 1.4128440366972477,
+      "grad_norm": 0.0531560480594635,
+      "learning_rate": 1.1982257481165547e-05,
+      "loss": 0.052,
+      "step": 154
+    },
+    {
+      "epoch": 1.4220183486238533,
+      "grad_norm": 0.0668332502245903,
+      "learning_rate": 1.1881854071210805e-05,
+      "loss": 0.0758,
+      "step": 155
+    },
+    {
+      "epoch": 1.4311926605504588,
+      "grad_norm": 0.06946220993995667,
+      "learning_rate": 1.1781253598229982e-05,
+      "loss": 0.0702,
+      "step": 156
+    },
+    {
+      "epoch": 1.4403669724770642,
+      "grad_norm": 0.06085884943604469,
+      "learning_rate": 1.1680466596851635e-05,
+      "loss": 0.0727,
+      "step": 157
+    },
+    {
+      "epoch": 1.4495412844036697,
+      "grad_norm": 0.04978602007031441,
+      "learning_rate": 1.1579503621237102e-05,
+      "loss": 0.077,
+      "step": 158
+    },
+    {
+      "epoch": 1.4587155963302751,
+      "grad_norm": 0.05822195112705231,
+      "learning_rate": 1.1478375243975298e-05,
+      "loss": 0.0769,
+      "step": 159
+    },
+    {
+      "epoch": 1.4678899082568808,
+      "grad_norm": 0.06906376779079437,
+      "learning_rate": 1.1377092054975586e-05,
+      "loss": 0.094,
+      "step": 160
+    },
+    {
+      "epoch": 1.4770642201834863,
+      "grad_norm": 0.0629039779305458,
+      "learning_rate": 1.1275664660358818e-05,
+      "loss": 0.0491,
+      "step": 161
+    },
+    {
+      "epoch": 1.4862385321100917,
+      "grad_norm": 0.05990944802761078,
+      "learning_rate": 1.1174103681346711e-05,
+      "loss": 0.0514,
+      "step": 162
+    },
+    {
+      "epoch": 1.4954128440366974,
+      "grad_norm": 0.055665481835603714,
+      "learning_rate": 1.1072419753149585e-05,
+      "loss": 0.0652,
+      "step": 163
+    },
+    {
+      "epoch": 1.5045871559633026,
+      "grad_norm": 0.04991630092263222,
+      "learning_rate": 1.0970623523852699e-05,
+      "loss": 0.0576,
+      "step": 164
+    },
+    {
+      "epoch": 1.5137614678899083,
+      "grad_norm": 0.052877284586429596,
+      "learning_rate": 1.0868725653301206e-05,
+      "loss": 0.075,
+      "step": 165
+    },
+    {
+      "epoch": 1.5229357798165137,
+      "grad_norm": 0.0749133750796318,
+      "learning_rate": 1.0766736811983864e-05,
+      "loss": 0.092,
+      "step": 166
+    },
+    {
+      "epoch": 1.5321100917431192,
+      "grad_norm": 0.050854723900556564,
+      "learning_rate": 1.066466767991567e-05,
+      "loss": 0.0282,
+      "step": 167
+    },
+    {
+      "epoch": 1.5412844036697249,
+      "grad_norm": 0.05233798176050186,
+      "learning_rate": 1.0562528945519463e-05,
+      "loss": 0.0688,
+      "step": 168
+    },
+    {
+      "epoch": 1.5412844036697249,
+      "eval_loss": 0.05367153137922287,
+      "eval_runtime": 43.3753,
+      "eval_samples_per_second": 3.988,
+      "eval_steps_per_second": 2.006,
+      "step": 168
+    },
+    {
+      "epoch": 1.5504587155963303,
+      "grad_norm": 0.06258448958396912,
+      "learning_rate": 1.0460331304506658e-05,
+      "loss": 0.0601,
+      "step": 169
+    },
+    {
+      "epoch": 1.5596330275229358,
+      "grad_norm": 0.05399218574166298,
+      "learning_rate": 1.0358085458757233e-05,
+      "loss": 0.0642,
+      "step": 170
+    },
+    {
+      "epoch": 1.5688073394495414,
+      "grad_norm": 0.052848368883132935,
+      "learning_rate": 1.0255802115199034e-05,
+      "loss": 0.054,
+      "step": 171
+    },
+    {
+      "epoch": 1.5779816513761467,
+      "grad_norm": 0.054908327758312225,
+      "learning_rate": 1.0153491984686595e-05,
+      "loss": 0.069,
+      "step": 172
+    },
+    {
+      "epoch": 1.5871559633027523,
+      "grad_norm": 0.08597759157419205,
+      "learning_rate": 1.0051165780879503e-05,
+      "loss": 0.0596,
+      "step": 173
+    },
+    {
+      "epoch": 1.5963302752293578,
+      "grad_norm": 0.06400232017040253,
+      "learning_rate": 9.9488342191205e-06,
+      "loss": 0.0959,
+      "step": 174
+    },
+    {
+      "epoch": 1.6055045871559632,
+      "grad_norm": 0.06301407516002655,
+      "learning_rate": 9.846508015313407e-06,
+      "loss": 0.0863,
+      "step": 175
+    },
+    {
+      "epoch": 1.614678899082569,
+      "grad_norm": 0.09353712201118469,
+      "learning_rate": 9.744197884800968e-06,
+      "loss": 0.0835,
+      "step": 176
+    },
+    {
+      "epoch": 1.6238532110091743,
+      "grad_norm": 0.14486946165561676,
+      "learning_rate": 9.64191454124277e-06,
+      "loss": 0.0786,
+      "step": 177
+    },
+    {
+      "epoch": 1.6330275229357798,
+      "grad_norm": 0.06457175314426422,
+      "learning_rate": 9.539668695493344e-06,
+      "loss": 0.0752,
+      "step": 178
+    },
+    {
+      "epoch": 1.6422018348623855,
+      "grad_norm": 0.059133633971214294,
+      "learning_rate": 9.43747105448054e-06,
+      "loss": 0.0747,
+      "step": 179
+    },
+    {
+      "epoch": 1.6513761467889907,
+      "grad_norm": 0.04722464829683304,
+      "learning_rate": 9.335332320084331e-06,
+      "loss": 0.0503,
+      "step": 180
+    },
+    {
+      "epoch": 1.6605504587155964,
+      "grad_norm": 0.06446046382188797,
+      "learning_rate": 9.233263188016138e-06,
+      "loss": 0.0794,
+      "step": 181
+    },
+    {
+      "epoch": 1.6697247706422018,
+      "grad_norm": 0.05856352299451828,
+      "learning_rate": 9.131274346698797e-06,
+      "loss": 0.0917,
+      "step": 182
+    },
+    {
+      "epoch": 1.6788990825688073,
+      "grad_norm": 0.050151705741882324,
+      "learning_rate": 9.029376476147303e-06,
+      "loss": 0.0534,
+      "step": 183
+    },
+    {
+      "epoch": 1.688073394495413,
+      "grad_norm": 0.11409275978803635,
+      "learning_rate": 8.927580246850418e-06,
+      "loss": 0.0579,
+      "step": 184
+    },
+    {
+      "epoch": 1.6972477064220184,
+      "grad_norm": 0.04147953912615776,
+      "learning_rate": 8.825896318653294e-06,
+      "loss": 0.0596,
+      "step": 185
+    },
+    {
+      "epoch": 1.7064220183486238,
+      "grad_norm": 0.06895549595355988,
+      "learning_rate": 8.724335339641185e-06,
+      "loss": 0.1267,
+      "step": 186
+    },
+    {
+      "epoch": 1.7155963302752295,
+      "grad_norm": 0.07597438991069794,
+      "learning_rate": 8.622907945024418e-06,
+      "loss": 0.0672,
+      "step": 187
+    },
+    {
+      "epoch": 1.7247706422018347,
+      "grad_norm": 0.0447760745882988,
+      "learning_rate": 8.521624756024706e-06,
+      "loss": 0.0619,
+      "step": 188
+    },
+    {
+      "epoch": 1.7339449541284404,
+      "grad_norm": 0.054984625428915024,
+      "learning_rate": 8.420496378762901e-06,
+      "loss": 0.0665,
+      "step": 189
+    },
+    {
+      "epoch": 1.7431192660550459,
+      "grad_norm": 0.06113699823617935,
+      "learning_rate": 8.319533403148368e-06,
+      "loss": 0.0755,
+      "step": 190
+    },
+    {
+      "epoch": 1.7522935779816513,
+      "grad_norm": 0.056827329099178314,
+      "learning_rate": 8.218746401770021e-06,
+      "loss": 0.0674,
+      "step": 191
+    },
+    {
+      "epoch": 1.761467889908257,
+      "grad_norm": 0.06809786707162857,
+      "learning_rate": 8.118145928789198e-06,
+      "loss": 0.0595,
+      "step": 192
+    },
+    {
+      "epoch": 1.7706422018348624,
+      "grad_norm": 0.05617869272828102,
+      "learning_rate": 8.017742518834454e-06,
+      "loss": 0.0823,
+      "step": 193
+    },
+    {
+      "epoch": 1.7798165137614679,
+      "grad_norm": 0.05785394459962845,
+      "learning_rate": 7.917546685898393e-06,
+      "loss": 0.099,
+      "step": 194
+    },
+    {
+      "epoch": 1.7889908256880735,
+      "grad_norm": 0.04695114120841026,
+      "learning_rate": 7.817568922236683e-06,
+      "loss": 0.0559,
+      "step": 195
+    },
+    {
+      "epoch": 1.7981651376146788,
+      "grad_norm": 0.04976266250014305,
+      "learning_rate": 7.717819697269322e-06,
+      "loss": 0.0424,
+      "step": 196
+    },
+    {
+      "epoch": 1.7981651376146788,
+      "eval_loss": 0.05340273305773735,
+      "eval_runtime": 43.3761,
+      "eval_samples_per_second": 3.988,
+      "eval_steps_per_second": 2.006,
+      "step": 196
+    },
+    {
+      "epoch": 1.8073394495412844,
+      "grad_norm": 0.04787492752075195,
+      "learning_rate": 7.618309456484309e-06,
+      "loss": 0.0439,
+      "step": 197
+    },
+    {
+      "epoch": 1.81651376146789,
+      "grad_norm": 0.048712193965911865,
+      "learning_rate": 7.519048620343825e-06,
+      "loss": 0.0714,
+      "step": 198
+    },
+    {
+      "epoch": 1.8256880733944953,
+      "grad_norm": 0.06686241924762726,
+      "learning_rate": 7.42004758319302e-06,
+      "loss": 0.0746,
+      "step": 199
+    },
+    {
+      "epoch": 1.834862385321101,
+      "grad_norm": 0.06182721257209778,
+      "learning_rate": 7.3213167121715514e-06,
+      "loss": 0.068,
+      "step": 200
+    },
+    {
+      "epoch": 1.8440366972477065,
+      "grad_norm": 0.05911610275506973,
+      "learning_rate": 7.222866346127952e-06,
+      "loss": 0.061,
+      "step": 201
+    },
+    {
+      "epoch": 1.853211009174312,
+      "grad_norm": 0.048602789640426636,
+      "learning_rate": 7.124706794536984e-06,
+      "loss": 0.0545,
+      "step": 202
+    },
+    {
+      "epoch": 1.8623853211009176,
+      "grad_norm": 0.09133545309305191,
+      "learning_rate": 7.026848336420053e-06,
+      "loss": 0.0579,
+      "step": 203
+    },
+    {
+      "epoch": 1.8715596330275228,
+      "grad_norm": 0.05401468276977539,
+      "learning_rate": 6.929301219268806e-06,
+      "loss": 0.0498,
+      "step": 204
+    },
+    {
+      "epoch": 1.8807339449541285,
+      "grad_norm": 0.057028841227293015,
+      "learning_rate": 6.8320756579720545e-06,
+      "loss": 0.1163,
+      "step": 205
+    },
+    {
+      "epoch": 1.889908256880734,
+      "grad_norm": 0.06496277451515198,
+      "learning_rate": 6.735181833746087e-06,
+      "loss": 0.0687,
+      "step": 206
+    },
+    {
+      "epoch": 1.8990825688073394,
+      "grad_norm": 0.04509355500340462,
+      "learning_rate": 6.638629893068516e-06,
+      "loss": 0.0833,
+      "step": 207
+    },
+    {
+      "epoch": 1.908256880733945,
+      "grad_norm": 0.05287083983421326,
+      "learning_rate": 6.542429946615774e-06,
+      "loss": 0.0506,
+      "step": 208
+    },
+    {
+      "epoch": 1.9174311926605505,
+      "grad_norm": 0.0586075522005558,
+      "learning_rate": 6.446592068204341e-06,
+      "loss": 0.0747,
+      "step": 209
+    },
+    {
+      "epoch": 1.926605504587156,
+      "grad_norm": 0.05395258963108063,
+      "learning_rate": 6.351126293735843e-06,
+      "loss": 0.0703,
+      "step": 210
+    },
+    {
+      "epoch": 1.9357798165137616,
+      "grad_norm": 0.04344907030463219,
+      "learning_rate": 6.256042620146119e-06,
+      "loss": 0.0445,
+      "step": 211
+    },
+    {
+      "epoch": 1.9449541284403669,
+      "grad_norm": 0.047881245613098145,
+      "learning_rate": 6.16135100435836e-06,
+      "loss": 0.063,
+      "step": 212
+    },
+    {
+      "epoch": 1.9541284403669725,
+      "grad_norm": 0.0735430121421814,
+      "learning_rate": 6.06706136224045e-06,
+      "loss": 0.0936,
+      "step": 213
+    },
+    {
+      "epoch": 1.963302752293578,
+      "grad_norm": 0.0707215815782547,
+      "learning_rate": 5.973183567566605e-06,
+      "loss": 0.0906,
+      "step": 214
+    },
+    {
+      "epoch": 1.9724770642201834,
+      "grad_norm": 0.05329213663935661,
+      "learning_rate": 5.879727450983412e-06,
+      "loss": 0.0665,
+      "step": 215
+    },
+    {
+      "epoch": 1.981651376146789,
+      "grad_norm": 0.04940500482916832,
+      "learning_rate": 5.786702798980388e-06,
+      "loss": 0.0649,
+      "step": 216
+    },
+    {
+      "epoch": 1.9908256880733946,
+      "grad_norm": 0.06716382503509521,
+      "learning_rate": 5.69411935286516e-06,
+      "loss": 0.0882,
+      "step": 217
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.06293365359306335,
+      "learning_rate": 5.601986807743388e-06,
+      "loss": 0.0649,
+      "step": 218
+    },
+    {
+      "epoch": 2.0091743119266057,
+      "grad_norm": 0.06269426643848419,
+      "learning_rate": 5.51031481150352e-06,
+      "loss": 0.0757,
+      "step": 219
+    },
+    {
+      "epoch": 2.018348623853211,
+      "grad_norm": 0.05706481635570526,
+      "learning_rate": 5.419112963806468e-06,
+      "loss": 0.0519,
+      "step": 220
+    },
+    {
+      "epoch": 2.0275229357798166,
+      "grad_norm": 0.05423853173851967,
+      "learning_rate": 5.328390815080381e-06,
+      "loss": 0.0828,
+      "step": 221
+    },
+    {
+      "epoch": 2.036697247706422,
+      "grad_norm": 0.04003476724028587,
+      "learning_rate": 5.238157865520539e-06,
+      "loss": 0.0431,
+      "step": 222
+    },
+    {
+      "epoch": 2.0458715596330275,
+      "grad_norm": 0.047263018786907196,
+      "learning_rate": 5.148423564094517e-06,
+      "loss": 0.0498,
+      "step": 223
+    },
+    {
+      "epoch": 2.055045871559633,
+      "grad_norm": 0.06567783653736115,
+      "learning_rate": 5.059197307552698e-06,
+      "loss": 0.086,
+      "step": 224
+    },
+    {
+      "epoch": 2.055045871559633,
+      "eval_loss": 0.05316643789410591,
+      "eval_runtime": 43.3865,
+      "eval_samples_per_second": 3.987,
+      "eval_steps_per_second": 2.005,
+      "step": 224
+    },
+    {
+      "epoch": 2.0642201834862384,
+      "grad_norm": 0.04639436677098274,
+      "learning_rate": 4.970488439444296e-06,
+      "loss": 0.0589,
+      "step": 225
+    },
+    {
+      "epoch": 2.073394495412844,
+      "grad_norm": 0.06990166008472443,
+      "learning_rate": 4.882306249138909e-06,
+      "loss": 0.0491,
+      "step": 226
+    },
+    {
+      "epoch": 2.0825688073394497,
+      "grad_norm": 0.05951413884758949,
+      "learning_rate": 4.7946599708537485e-06,
+      "loss": 0.0736,
+      "step": 227
+    },
+    {
+      "epoch": 2.091743119266055,
+      "grad_norm": 0.04998117685317993,
+      "learning_rate": 4.707558782686677e-06,
+      "loss": 0.073,
+      "step": 228
+    },
+    {
+      "epoch": 2.1009174311926606,
+      "grad_norm": 0.05194695666432381,
+      "learning_rate": 4.621011805655093e-06,
+      "loss": 0.0797,
+      "step": 229
+    },
+    {
+      "epoch": 2.1100917431192663,
+      "grad_norm": 0.058086711913347244,
+      "learning_rate": 4.535028102740785e-06,
+      "loss": 0.0612,
+      "step": 230
+    },
+    {
+      "epoch": 2.1192660550458715,
+      "grad_norm": 0.05054466798901558,
+      "learning_rate": 4.449616677940904e-06,
+      "loss": 0.0623,
+      "step": 231
+    },
+    {
+      "epoch": 2.128440366972477,
+      "grad_norm": 0.05400394648313522,
+      "learning_rate": 4.364786475325072e-06,
+      "loss": 0.0778,
+      "step": 232
+    },
+    {
+      "epoch": 2.1376146788990824,
+      "grad_norm": 0.049106135964393616,
+      "learning_rate": 4.280546378098792e-06,
+      "loss": 0.0673,
+      "step": 233
+    },
+    {
+      "epoch": 2.146788990825688,
+      "grad_norm": 0.05438579246401787,
+      "learning_rate": 4.196905207673201e-06,
+      "loss": 0.071,
+      "step": 234
+    },
+    {
+      "epoch": 2.1559633027522938,
+      "grad_norm": 0.05540682002902031,
+      "learning_rate": 4.113871722741337e-06,
+      "loss": 0.0691,
+      "step": 235
+    },
+    {
+      "epoch": 2.165137614678899,
+      "grad_norm": 0.07121812552213669,
+      "learning_rate": 4.031454618360945e-06,
+      "loss": 0.0859,
+      "step": 236
+    },
+    {
+      "epoch": 2.1743119266055047,
+      "grad_norm": 0.06990870833396912,
+      "learning_rate": 3.949662525043935e-06,
+      "loss": 0.0808,
+      "step": 237
+    },
+    {
+      "epoch": 2.18348623853211,
+      "grad_norm": 0.04688805714249611,
+      "learning_rate": 3.868504007852641e-06,
+      "loss": 0.0552,
+      "step": 238
+    },
+    {
+      "epoch": 2.1926605504587156,
+      "grad_norm": 0.05368739366531372,
+      "learning_rate": 3.7879875655029018e-06,
+      "loss": 0.0708,
+      "step": 239
+    },
+    {
+      "epoch": 2.2018348623853212,
+      "grad_norm": 0.05669724941253662,
+      "learning_rate": 3.7081216294740773e-06,
+      "loss": 0.0714,
+      "step": 240
+    },
+    {
+      "epoch": 2.2110091743119265,
+      "grad_norm": 0.07314234972000122,
+      "learning_rate": 3.628914563126156e-06,
+      "loss": 0.0842,
+      "step": 241
+    },
+    {
+      "epoch": 2.220183486238532,
+      "grad_norm": 0.06005195155739784,
+      "learning_rate": 3.5503746608239487e-06,
+      "loss": 0.0637,
+      "step": 242
+    },
+    {
+      "epoch": 2.229357798165138,
+      "grad_norm": 0.05186507850885391,
+      "learning_rate": 3.472510147068515e-06,
+      "loss": 0.0606,
+      "step": 243
+    },
+    {
+      "epoch": 2.238532110091743,
+      "grad_norm": 0.058381304144859314,
+      "learning_rate": 3.3953291756359354e-06,
+      "loss": 0.0626,
+      "step": 244
+    },
+    {
+      "epoch": 2.2477064220183487,
+      "grad_norm": 0.05521458014845848,
+      "learning_rate": 3.3188398287234504e-06,
+      "loss": 0.0628,
+      "step": 245
+    },
+    {
+      "epoch": 2.2568807339449544,
+      "grad_norm": 0.05869848653674126,
+      "learning_rate": 3.243050116103128e-06,
+      "loss": 0.0448,
+      "step": 246
+    },
+    {
+      "epoch": 2.2660550458715596,
+      "grad_norm": 0.07123305648565292,
+      "learning_rate": 3.1679679742830806e-06,
+      "loss": 0.0991,
+      "step": 247
+    },
+    {
+      "epoch": 2.2752293577981653,
+      "grad_norm": 0.05694476515054703,
+      "learning_rate": 3.0936012656763937e-06,
+      "loss": 0.1169,
+      "step": 248
+    },
+    {
+      "epoch": 2.2844036697247705,
+      "grad_norm": 0.04774909466505051,
+      "learning_rate": 3.019957777777788e-06,
+      "loss": 0.0655,
+      "step": 249
+    },
+    {
+      "epoch": 2.293577981651376,
+      "grad_norm": 0.05831342935562134,
+      "learning_rate": 2.9470452223481206e-06,
+      "loss": 0.0618,
+      "step": 250
+    },
+    {
+      "epoch": 2.302752293577982,
+      "grad_norm": 0.08326123654842377,
+      "learning_rate": 2.8748712346068464e-06,
+      "loss": 0.0923,
+      "step": 251
+    },
+    {
+      "epoch": 2.311926605504587,
+      "grad_norm": 0.06080171838402748,
+      "learning_rate": 2.8034433724324716e-06,
+      "loss": 0.0759,
+      "step": 252
+    },
+    {
+      "epoch": 2.311926605504587,
+      "eval_loss": 0.052994709461927414,
+      "eval_runtime": 43.3989,
+      "eval_samples_per_second": 3.986,
+      "eval_steps_per_second": 2.005,
+      "step": 252
+    },
+    {
+      "epoch": 2.3211009174311927,
+      "grad_norm": 0.06857974827289581,
+      "learning_rate": 2.7327691155710978e-06,
+      "loss": 0.0791,
+      "step": 253
+    },
+    {
+      "epoch": 2.330275229357798,
+      "grad_norm": 0.08638834208250046,
+      "learning_rate": 2.6628558648531845e-06,
+      "loss": 0.0894,
+      "step": 254
+    },
+    {
+      "epoch": 2.3394495412844036,
+      "grad_norm": 0.07400725036859512,
+      "learning_rate": 2.593710941418537e-06,
+      "loss": 0.0624,
+      "step": 255
+    },
+    {
+      "epoch": 2.3486238532110093,
+      "grad_norm": 0.05066663771867752,
+      "learning_rate": 2.525341585949662e-06,
+      "loss": 0.0755,
+      "step": 256
+    },
+    {
+      "epoch": 2.3577981651376145,
+      "grad_norm": 0.051476072520017624,
+      "learning_rate": 2.4577549579135318e-06,
+      "loss": 0.074,
+      "step": 257
+    },
+    {
+      "epoch": 2.36697247706422,
+      "grad_norm": 0.05693186819553375,
+      "learning_rate": 2.3909581348118803e-06,
+      "loss": 0.049,
+      "step": 258
+    },
+    {
+      "epoch": 2.376146788990826,
+      "grad_norm": 0.043929580599069595,
+      "learning_rate": 2.324958111440051e-06,
+      "loss": 0.0558,
+      "step": 259
+    },
+    {
+      "epoch": 2.385321100917431,
+      "grad_norm": 0.06177612394094467,
+      "learning_rate": 2.259761799154516e-06,
+      "loss": 0.0634,
+      "step": 260
+    },
+    {
+      "epoch": 2.3944954128440368,
+      "grad_norm": 0.08901379257440567,
+      "learning_rate": 2.195376025149156e-06,
+      "loss": 0.0552,
+      "step": 261
+    },
+    {
+      "epoch": 2.4036697247706424,
+      "grad_norm": 0.059478871524333954,
+      "learning_rate": 2.1318075317403152e-06,
+      "loss": 0.0834,
+      "step": 262
+    },
+    {
+      "epoch": 2.4128440366972477,
+      "grad_norm": 0.14992526173591614,
+      "learning_rate": 2.069062975660765e-06,
+      "loss": 0.0582,
+      "step": 263
+    },
+    {
+      "epoch": 2.4220183486238533,
+      "grad_norm": 0.04817449301481247,
+      "learning_rate": 2.0071489273626376e-06,
+      "loss": 0.0547,
+      "step": 264
+    },
+    {
+      "epoch": 2.4311926605504586,
+      "grad_norm": 0.08196448534727097,
+      "learning_rate": 1.946071870329377e-06,
+      "loss": 0.078,
+      "step": 265
+    },
+    {
+      "epoch": 2.4403669724770642,
+      "grad_norm": 0.07558903098106384,
+      "learning_rate": 1.885838200396808e-06,
+      "loss": 0.0507,
+      "step": 266
+    },
+    {
+      "epoch": 2.44954128440367,
+      "grad_norm": 0.061492372304201126,
+      "learning_rate": 1.826454225083375e-06,
+      "loss": 0.0526,
+      "step": 267
+    },
+    {
+      "epoch": 2.458715596330275,
+      "grad_norm": 0.04717002436518669,
+      "learning_rate": 1.7679261629296408e-06,
+      "loss": 0.05,
+      "step": 268
+    },
+    {
+      "epoch": 2.467889908256881,
+      "grad_norm": 0.050578705966472626,
+      "learning_rate": 1.7102601428470988e-06,
+      "loss": 0.0694,
+      "step": 269
+    },
+    {
+      "epoch": 2.477064220183486,
+      "grad_norm": 0.06575262546539307,
+      "learning_rate": 1.6534622034763558e-06,
+      "loss": 0.0537,
+      "step": 270
+    },
+    {
+      "epoch": 2.4862385321100917,
+      "grad_norm": 0.0549924410879612,
+      "learning_rate": 1.5975382925547966e-06,
+      "loss": 0.0802,
+      "step": 271
+    },
+    {
+      "epoch": 2.4954128440366974,
+      "grad_norm": 0.06130588427186012,
+      "learning_rate": 1.5424942662937436e-06,
+      "loss": 0.0766,
+      "step": 272
+    },
+    {
+      "epoch": 2.5045871559633026,
+      "grad_norm": 0.07862205803394318,
+      "learning_rate": 1.4883358887652044e-06,
+      "loss": 0.0612,
+      "step": 273
+    },
+    {
+      "epoch": 2.5137614678899083,
+      "grad_norm": 0.04936962202191353,
+      "learning_rate": 1.4350688312982864e-06,
+      "loss": 0.0556,
+      "step": 274
+    },
+    {
+      "epoch": 2.522935779816514,
+      "grad_norm": 0.06410589069128036,
+      "learning_rate": 1.3826986718852952e-06,
+      "loss": 0.0483,
+      "step": 275
+    },
+    {
+      "epoch": 2.532110091743119,
+      "grad_norm": 0.09402082115411758,
+      "learning_rate": 1.3312308945976348e-06,
+      "loss": 0.1031,
+      "step": 276
+    },
+    {
+      "epoch": 2.541284403669725,
+      "grad_norm": 0.052867498248815536,
+      "learning_rate": 1.2806708890115138e-06,
+      "loss": 0.065,
+      "step": 277
+    },
+    {
+      "epoch": 2.5504587155963305,
+      "grad_norm": 0.08837206661701202,
+      "learning_rate": 1.2310239496435749e-06,
+      "loss": 0.095,
+      "step": 278
+    },
+    {
+      "epoch": 2.5596330275229358,
+      "grad_norm": 0.08973362296819687,
+      "learning_rate": 1.1822952753964667e-06,
+      "loss": 0.0765,
+      "step": 279
+    },
+    {
+      "epoch": 2.5688073394495414,
+      "grad_norm": 0.061795495450496674,
+      "learning_rate": 1.134489969014414e-06,
+      "loss": 0.0583,
+      "step": 280
+    },
+    {
+      "epoch": 2.5688073394495414,
+      "eval_loss": 0.05294761061668396,
+      "eval_runtime": 43.223,
+      "eval_samples_per_second": 4.002,
+      "eval_steps_per_second": 2.013,
+      "step": 280
+    },
+    {
+      "epoch": 2.5779816513761467,
+      "grad_norm": 0.047972485423088074,
+      "learning_rate": 1.087613036548888e-06,
+      "loss": 0.048,
+      "step": 281
+    },
+    {
+      "epoch": 2.5871559633027523,
+      "grad_norm": 0.06896362453699112,
+      "learning_rate": 1.0416693868343796e-06,
+      "loss": 0.0771,
+      "step": 282
+    },
+    {
+      "epoch": 2.5963302752293576,
+      "grad_norm": 0.06132780387997627,
+      "learning_rate": 9.966638309743481e-07,
+      "loss": 0.0854,
+      "step": 283
+    },
+    {
+      "epoch": 2.6055045871559632,
+      "grad_norm": 0.06309553980827332,
+      "learning_rate": 9.52601081837431e-07,
+      "loss": 0.0827,
+      "step": 284
+    },
+    {
+      "epoch": 2.614678899082569,
+      "grad_norm": 0.08898341655731201,
+      "learning_rate": 9.094857535639157e-07,
+      "loss": 0.0727,
+      "step": 285
+    },
+    {
+      "epoch": 2.623853211009174,
+      "grad_norm": 0.05615299567580223,
+      "learning_rate": 8.673223610825532e-07,
+      "loss": 0.0827,
+      "step": 286
+    },
+    {
+      "epoch": 2.63302752293578,
+      "grad_norm": 0.06234830617904663,
+      "learning_rate": 8.261153196377814e-07,
+      "loss": 0.0772,
+      "step": 287
+    },
+    {
+      "epoch": 2.6422018348623855,
+      "grad_norm": 0.057416193187236786,
+      "learning_rate": 7.858689443273548e-07,
+      "loss": 0.0726,
+      "step": 288
+    },
+    {
+      "epoch": 2.6513761467889907,
+      "grad_norm": 0.056388452649116516,
+      "learning_rate": 7.465874496504944e-07,
+      "loss": 0.0881,
+      "step": 289
+    },
+    {
+      "epoch": 2.6605504587155964,
+      "grad_norm": 0.05161774531006813,
+      "learning_rate": 7.082749490665353e-07,
+      "loss": 0.0447,
+      "step": 290
+    },
+    {
+      "epoch": 2.669724770642202,
+      "grad_norm": 0.047958966344594955,
+      "learning_rate": 6.709354545641989e-07,
+      "loss": 0.0772,
+      "step": 291
+    },
+    {
+      "epoch": 2.6788990825688073,
+      "grad_norm": 0.0640062615275383,
+      "learning_rate": 6.345728762414504e-07,
+      "loss": 0.0607,
+      "step": 292
+    },
+    {
+      "epoch": 2.688073394495413,
+      "grad_norm": 0.05299694091081619,
+      "learning_rate": 5.99191021896055e-07,
+      "loss": 0.047,
+      "step": 293
+    },
+    {
+      "epoch": 2.6972477064220186,
+      "grad_norm": 0.057945024222135544,
+      "learning_rate": 5.647935966268225e-07,
+      "loss": 0.0731,
+      "step": 294
+    },
+    {
+      "epoch": 2.706422018348624,
+      "grad_norm": 0.05141222104430199,
+      "learning_rate": 5.313842024456306e-07,
+      "loss": 0.039,
+      "step": 295
+    },
+    {
+      "epoch": 2.7155963302752295,
+      "grad_norm": 0.08650866150856018,
+      "learning_rate": 4.98966337900224e-07,
+      "loss": 0.0551,
+      "step": 296
+    },
+    {
+      "epoch": 2.7247706422018347,
+      "grad_norm": 0.13347071409225464,
+      "learning_rate": 4.6754339770785474e-07,
+      "loss": 0.0619,
+      "step": 297
+    },
+    {
+      "epoch": 2.7339449541284404,
+      "grad_norm": 0.07346609234809875,
+      "learning_rate": 4.3711867239980335e-07,
+      "loss": 0.0423,
+      "step": 298
+    },
+    {
+      "epoch": 2.7431192660550456,
+      "grad_norm": 0.05696272850036621,
+      "learning_rate": 4.076953479767964e-07,
+      "loss": 0.0903,
+      "step": 299
+    },
+    {
+      "epoch": 2.7522935779816513,
+      "grad_norm": 0.058090586215257645,
+      "learning_rate": 3.792765055753755e-07,
+      "loss": 0.0994,
+      "step": 300
+    },
+    {
+      "epoch": 2.761467889908257,
+      "grad_norm": 0.06176576018333435,
+      "learning_rate": 3.5186512114525283e-07,
+      "loss": 0.1243,
+      "step": 301
+    },
+    {
+      "epoch": 2.770642201834862,
+      "grad_norm": 0.07493139058351517,
+      "learning_rate": 3.25464065137675e-07,
+      "loss": 0.0584,
+      "step": 302
+    },
+    {
+      "epoch": 2.779816513761468,
+      "grad_norm": 0.04847017675638199,
+      "learning_rate": 3.0007610220483927e-07,
+      "loss": 0.0652,
+      "step": 303
+    },
+    {
+      "epoch": 2.7889908256880735,
+      "grad_norm": 0.058301348239183426,
+      "learning_rate": 2.757038909103793e-07,
+      "loss": 0.0708,
+      "step": 304
+    },
+    {
+      "epoch": 2.7981651376146788,
+      "grad_norm": 0.046219125390052795,
+      "learning_rate": 2.523499834509724e-07,
+      "loss": 0.0495,
+      "step": 305
+    },
+    {
+      "epoch": 2.8073394495412844,
+      "grad_norm": 0.051395233720541,
+      "learning_rate": 2.3001682538908333e-07,
+      "loss": 0.0954,
+      "step": 306
+    },
+    {
+      "epoch": 2.81651376146789,
+      "grad_norm": 0.0680239349603653,
+      "learning_rate": 2.0870675539686024e-07,
+      "loss": 0.0717,
+      "step": 307
+    },
+    {
+      "epoch": 2.8256880733944953,
+      "grad_norm": 0.058481365442276,
+      "learning_rate": 1.884220050112462e-07,
+      "loss": 0.1087,
+      "step": 308
+    },
+    {
+      "epoch": 2.8256880733944953,
+      "eval_loss": 0.05293623358011246,
+      "eval_runtime": 43.297,
+      "eval_samples_per_second": 3.996,
+      "eval_steps_per_second": 2.009,
+      "step": 308
+    },
+    {
+      "epoch": 2.834862385321101,
+      "grad_norm": 0.06916595250368118,
+      "learning_rate": 1.691646984002937e-07,
+      "loss": 0.052,
+      "step": 309
+    },
+    {
+      "epoch": 2.8440366972477067,
+      "grad_norm": 0.06048694625496864,
+      "learning_rate": 1.5093685214072173e-07,
+      "loss": 0.0508,
+      "step": 310
+    },
+    {
+      "epoch": 2.853211009174312,
+      "grad_norm": 0.0559132881462574,
+      "learning_rate": 1.3374037500675452e-07,
+      "loss": 0.0992,
+      "step": 311
+    },
+    {
+      "epoch": 2.8623853211009176,
+      "grad_norm": 0.05447855591773987,
+      "learning_rate": 1.1757706777023592e-07,
+      "loss": 0.101,
+      "step": 312
+    },
+    {
+      "epoch": 2.871559633027523,
+      "grad_norm": 0.05133409798145294,
+      "learning_rate": 1.024486230120525e-07,
+      "loss": 0.0481,
+      "step": 313
+    },
+    {
+      "epoch": 2.8807339449541285,
+      "grad_norm": 0.05679594352841377,
+      "learning_rate": 8.835662494489638e-08,
+      "loss": 0.0542,
+      "step": 314
+    },
+    {
+      "epoch": 2.8899082568807337,
+      "grad_norm": 0.08849138766527176,
+      "learning_rate": 7.530254924736691e-08,
+      "loss": 0.0529,
+      "step": 315
+    },
+    {
+      "epoch": 2.8990825688073394,
+      "grad_norm": 0.05840228870511055,
+      "learning_rate": 6.32877629094475e-08,
+      "loss": 0.0509,
+      "step": 316
+    },
+    {
+      "epoch": 2.908256880733945,
+      "grad_norm": 0.05628720670938492,
+      "learning_rate": 5.231352408934687e-08,
+      "loss": 0.0684,
+      "step": 317
+    },
+    {
+      "epoch": 2.9174311926605503,
+      "grad_norm": 0.09971769899129868,
+      "learning_rate": 4.2380981981759994e-08,
+      "loss": 0.0992,
+      "step": 318
+    },
+    {
+      "epoch": 2.926605504587156,
+      "grad_norm": 0.05017199367284775,
+      "learning_rate": 3.349117669751767e-08,
+      "loss": 0.0506,
+      "step": 319
+    },
+    {
+      "epoch": 2.9357798165137616,
+      "grad_norm": 0.041759029030799866,
+      "learning_rate": 2.5645039154675867e-08,
+      "loss": 0.0475,
+      "step": 320
+    },
+    {
+      "epoch": 2.944954128440367,
+      "grad_norm": 0.04425744712352753,
+      "learning_rate": 1.8843390981024835e-08,
+      "loss": 0.0421,
+      "step": 321
+    },
+    {
+      "epoch": 2.9541284403669725,
+      "grad_norm": 0.04827776178717613,
+      "learning_rate": 1.3086944428060132e-08,
+      "loss": 0.0628,
+      "step": 322
+    },
+    {
+      "epoch": 2.963302752293578,
+      "grad_norm": 0.05134027451276779,
+      "learning_rate": 8.376302296387862e-09,
+      "loss": 0.068,
+      "step": 323
+    },
+    {
+      "epoch": 2.9724770642201834,
+      "grad_norm": 0.04932725057005882,
+      "learning_rate": 4.711957872606254e-09,
+      "loss": 0.0787,
+      "step": 324
+    },
+    {
+      "epoch": 2.981651376146789,
+      "grad_norm": 0.04113076254725456,
+      "learning_rate": 2.0942948776481175e-09,
+      "loss": 0.0384,
+      "step": 325
+    },
+    {
+      "epoch": 2.9908256880733948,
+      "grad_norm": 0.06910436600446701,
+      "learning_rate": 5.23587426601857e-10,
+      "loss": 0.0805,
+      "step": 326
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.042928002774715424,
+      "learning_rate": 0.0,
+      "loss": 0.0588,
+      "step": 327
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 327,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 28,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.0294089094536888e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-327/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8691334e7ff9d485bc39601a29a4096723c23e5fb7323cdb19a40a1c9c993c02
+size 6520

config.json

@@ -0,0 +1,48 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "bigcode/starcoder2-15b",
+  "architectures": [
+    "Starcoder2ForCausalLM"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "embedding_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_size": 6144,
+  "initializer_range": 0.01275,
+  "intermediate_size": 24576,
+  "max_position_embeddings": 16384,
+  "mlp_type": "default",
+  "model_type": "starcoder2",
+  "norm_epsilon": 1e-05,
+  "norm_type": "layer_norm",
+  "num_attention_heads": 48,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 4,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": true,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "residual_dropout": 0.1,
+  "rope_scaling": null,
+  "rope_theta": 100000,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0",
+  "use_bias": true,
+  "use_cache": false,
+  "vocab_size": 49152
+}

special_tokens_map.json

@@ -0,0 +1,70 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,358 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "37": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}