Upload folder using huggingface_hub

full-lora/README.md

@@ -0,0 +1,62 @@
+---
+base_model: meta-llama/Llama-2-7b-chat-hf
+library_name: peft
+model_name: llama2-7b-chat-extended-refusal-lora
+tags:
+- base_model:adapter:meta-llama/Llama-2-7b-chat-hf
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for llama2-7b-chat-extended-refusal-lora
+
+This model is a fine-tuned version of [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 1.0.0
+- Transformers: 4.57.6
+- Pytorch: 2.7.0+cu126
+- Datasets: 4.8.4
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

full-lora/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

full-lora/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23038c1e179be1b0cef2b41a2699c1eb17948c75072dcc812915cb90731e30eb
+size 639691872

full-lora/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\n' + system_message + '\n<</SYS>>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}

full-lora/checkpoint-1250/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-2-7b-chat-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-2-7b-chat-hf
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

full-lora/checkpoint-1250/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

full-lora/checkpoint-1250/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee2785be37d311a437ce764c3d902243e4d7a149b600034676da8c4c351d0c8c
+size 639691872

full-lora/checkpoint-1250/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\n' + system_message + '\n<</SYS>>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}

full-lora/checkpoint-1250/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:278e790cbd9dce193ebee8bb0eda5ade1faa89015d9de82de18905562b523dd7
+size 1279641507

full-lora/checkpoint-1250/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:384f52019480d29d41ab55cdcb5ee4c7187d8318c7e537d7498c12651e9404d4
+size 14645

full-lora/checkpoint-1250/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44079d3406189dd8c0b1f4b8cb07099f44e3631f418b40c9403687cbf7c990a3
+size 1465

full-lora/checkpoint-1250/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

full-lora/checkpoint-1250/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

full-lora/checkpoint-1250/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

full-lora/checkpoint-1250/trainer_state.json

@@ -0,0 +1,1284 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.9034420549869537,
+      "epoch": 0.008,
+      "grad_norm": 1.8304097652435303,
+      "learning_rate": 7.964601769911503e-08,
+      "loss": 1.9907,
+      "mean_token_accuracy": 0.6042250197380781,
+      "num_tokens": 15409.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.9475356195122003,
+      "epoch": 0.016,
+      "grad_norm": 1.5987296104431152,
+      "learning_rate": 1.68141592920354e-07,
+      "loss": 2.0397,
+      "mean_token_accuracy": 0.5890234299004078,
+      "num_tokens": 30650.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.9629276197403669,
+      "epoch": 0.024,
+      "grad_norm": 2.1185450553894043,
+      "learning_rate": 2.5663716814159294e-07,
+      "loss": 2.0312,
+      "mean_token_accuracy": 0.5870847202837467,
+      "num_tokens": 45561.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.9790993589907885,
+      "epoch": 0.032,
+      "grad_norm": 2.317858934402466,
+      "learning_rate": 3.451327433628318e-07,
+      "loss": 2.0456,
+      "mean_token_accuracy": 0.5804481774568557,
+      "num_tokens": 59568.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.922547810524702,
+      "epoch": 0.04,
+      "grad_norm": 1.5205438137054443,
+      "learning_rate": 4.3362831858407076e-07,
+      "loss": 1.9502,
+      "mean_token_accuracy": 0.5938452776521445,
+      "num_tokens": 75405.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.03425248041749,
+      "epoch": 0.048,
+      "grad_norm": 2.2233166694641113,
+      "learning_rate": 5.221238938053097e-07,
+      "loss": 2.1455,
+      "mean_token_accuracy": 0.5820784538984298,
+      "num_tokens": 88641.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.9562045603990554,
+      "epoch": 0.056,
+      "grad_norm": 1.6535303592681885,
+      "learning_rate": 6.106194690265486e-07,
+      "loss": 1.9628,
+      "mean_token_accuracy": 0.5903334401547908,
+      "num_tokens": 104448.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.9437448482960462,
+      "epoch": 0.064,
+      "grad_norm": 2.1216635704040527,
+      "learning_rate": 6.991150442477876e-07,
+      "loss": 2.0221,
+      "mean_token_accuracy": 0.5954494498670101,
+      "num_tokens": 119162.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.9662446774542331,
+      "epoch": 0.072,
+      "grad_norm": 1.6397205591201782,
+      "learning_rate": 7.876106194690266e-07,
+      "loss": 1.8199,
+      "mean_token_accuracy": 0.6050666151568294,
+      "num_tokens": 136337.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.9452362310141325,
+      "epoch": 0.08,
+      "grad_norm": 2.1384687423706055,
+      "learning_rate": 8.761061946902655e-07,
+      "loss": 2.0152,
+      "mean_token_accuracy": 0.6010695319622755,
+      "num_tokens": 150710.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.077597926557064,
+      "epoch": 0.088,
+      "grad_norm": 2.1343936920166016,
+      "learning_rate": 9.646017699115042e-07,
+      "loss": 2.0805,
+      "mean_token_accuracy": 0.5725633725523949,
+      "num_tokens": 163196.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.0407875191420317,
+      "epoch": 0.096,
+      "grad_norm": 2.0486209392547607,
+      "learning_rate": 9.983502886994776e-07,
+      "loss": 1.9709,
+      "mean_token_accuracy": 0.5931926343590022,
+      "num_tokens": 177368.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.0551968751475216,
+      "epoch": 0.104,
+      "grad_norm": 1.6275029182434082,
+      "learning_rate": 9.956007698652736e-07,
+      "loss": 1.8687,
+      "mean_token_accuracy": 0.6001733940094709,
+      "num_tokens": 191720.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.0219032436609268,
+      "epoch": 0.112,
+      "grad_norm": 1.6011073589324951,
+      "learning_rate": 9.928512510310695e-07,
+      "loss": 1.7826,
+      "mean_token_accuracy": 0.6063280884176493,
+      "num_tokens": 207354.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.057712158560753,
+      "epoch": 0.12,
+      "grad_norm": 1.6710939407348633,
+      "learning_rate": 9.901017321968655e-07,
+      "loss": 1.6872,
+      "mean_token_accuracy": 0.6104031000286341,
+      "num_tokens": 223850.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.1023272581398487,
+      "epoch": 0.128,
+      "grad_norm": 1.8599427938461304,
+      "learning_rate": 9.873522133626614e-07,
+      "loss": 1.6716,
+      "mean_token_accuracy": 0.6082586232572794,
+      "num_tokens": 239768.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1792589366436004,
+      "epoch": 0.136,
+      "grad_norm": 1.7112274169921875,
+      "learning_rate": 9.846026945284574e-07,
+      "loss": 1.7211,
+      "mean_token_accuracy": 0.606986602395773,
+      "num_tokens": 254892.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.2239510141313077,
+      "epoch": 0.144,
+      "grad_norm": 1.8953965902328491,
+      "learning_rate": 9.818531756942533e-07,
+      "loss": 1.7478,
+      "mean_token_accuracy": 0.6006653252989054,
+      "num_tokens": 269298.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1604003377258778,
+      "epoch": 0.152,
+      "grad_norm": 1.2709853649139404,
+      "learning_rate": 9.791036568600495e-07,
+      "loss": 1.5892,
+      "mean_token_accuracy": 0.6230274979025126,
+      "num_tokens": 283711.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1698237597942351,
+      "epoch": 0.16,
+      "grad_norm": 1.2409337759017944,
+      "learning_rate": 9.763541380258455e-07,
+      "loss": 1.4959,
+      "mean_token_accuracy": 0.6206645734608174,
+      "num_tokens": 301123.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.3337292484939098,
+      "epoch": 0.168,
+      "grad_norm": 1.0640740394592285,
+      "learning_rate": 9.736046191916414e-07,
+      "loss": 1.6387,
+      "mean_token_accuracy": 0.6070953067392111,
+      "num_tokens": 315342.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.2350904777646066,
+      "epoch": 0.176,
+      "grad_norm": 1.3057981729507446,
+      "learning_rate": 9.708551003574374e-07,
+      "loss": 1.5271,
+      "mean_token_accuracy": 0.6268062952905893,
+      "num_tokens": 332035.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.2457694321870805,
+      "epoch": 0.184,
+      "grad_norm": 1.434810996055603,
+      "learning_rate": 9.681055815232333e-07,
+      "loss": 1.4719,
+      "mean_token_accuracy": 0.627309649437666,
+      "num_tokens": 348533.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.3725072897970676,
+      "epoch": 0.192,
+      "grad_norm": 1.203586459159851,
+      "learning_rate": 9.653560626890293e-07,
+      "loss": 1.4614,
+      "mean_token_accuracy": 0.618980173021555,
+      "num_tokens": 363915.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.3240537498146296,
+      "epoch": 0.2,
+      "grad_norm": 1.5373510122299194,
+      "learning_rate": 9.626065438548252e-07,
+      "loss": 1.4521,
+      "mean_token_accuracy": 0.6376201532781124,
+      "num_tokens": 380021.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.4129172183573246,
+      "epoch": 0.208,
+      "grad_norm": 1.0626189708709717,
+      "learning_rate": 9.598570250206214e-07,
+      "loss": 1.4791,
+      "mean_token_accuracy": 0.639534817636013,
+      "num_tokens": 394415.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.4732031315565108,
+      "epoch": 0.216,
+      "grad_norm": 1.2482506036758423,
+      "learning_rate": 9.571075061864174e-07,
+      "loss": 1.5304,
+      "mean_token_accuracy": 0.6208007004112005,
+      "num_tokens": 408643.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.4145490050315856,
+      "epoch": 0.224,
+      "grad_norm": 1.069368600845337,
+      "learning_rate": 9.543579873522133e-07,
+      "loss": 1.3775,
+      "mean_token_accuracy": 0.6412477158010006,
+      "num_tokens": 423952.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.4205268509685993,
+      "epoch": 0.232,
+      "grad_norm": 0.8400747776031494,
+      "learning_rate": 9.516084685180094e-07,
+      "loss": 1.4327,
+      "mean_token_accuracy": 0.6374875742942094,
+      "num_tokens": 438452.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.4074747256934643,
+      "epoch": 0.24,
+      "grad_norm": 1.1713480949401855,
+      "learning_rate": 9.488589496838053e-07,
+      "loss": 1.4076,
+      "mean_token_accuracy": 0.640987902879715,
+      "num_tokens": 453417.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.4747156761586666,
+      "epoch": 0.248,
+      "grad_norm": 0.9282711744308472,
+      "learning_rate": 9.461094308496013e-07,
+      "loss": 1.4397,
+      "mean_token_accuracy": 0.6315288979560136,
+      "num_tokens": 467659.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.407252886891365,
+      "epoch": 0.256,
+      "grad_norm": 0.6602013111114502,
+      "learning_rate": 9.433599120153973e-07,
+      "loss": 1.3759,
+      "mean_token_accuracy": 0.6466614466160536,
+      "num_tokens": 484442.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.4545464389026166,
+      "epoch": 0.264,
+      "grad_norm": 1.206853985786438,
+      "learning_rate": 9.406103931811933e-07,
+      "loss": 1.4166,
+      "mean_token_accuracy": 0.623085155710578,
+      "num_tokens": 498395.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.4992636159062385,
+      "epoch": 0.272,
+      "grad_norm": 0.8671019673347473,
+      "learning_rate": 9.378608743469893e-07,
+      "loss": 1.3696,
+      "mean_token_accuracy": 0.6376510679721832,
+      "num_tokens": 513410.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.4165021255612373,
+      "epoch": 0.28,
+      "grad_norm": 0.7934819459915161,
+      "learning_rate": 9.351113555127852e-07,
+      "loss": 1.2903,
+      "mean_token_accuracy": 0.6567342653870583,
+      "num_tokens": 529739.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.5230652637779714,
+      "epoch": 0.288,
+      "grad_norm": 0.7465261220932007,
+      "learning_rate": 9.323618366785813e-07,
+      "loss": 1.369,
+      "mean_token_accuracy": 0.6413242511451245,
+      "num_tokens": 543258.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.4946719333529472,
+      "epoch": 0.296,
+      "grad_norm": 0.7156000733375549,
+      "learning_rate": 9.296123178443772e-07,
+      "loss": 1.3848,
+      "mean_token_accuracy": 0.6313502866774797,
+      "num_tokens": 558401.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.433413552492857,
+      "epoch": 0.304,
+      "grad_norm": 0.8262195587158203,
+      "learning_rate": 9.268627990101732e-07,
+      "loss": 1.346,
+      "mean_token_accuracy": 0.6439653303474188,
+      "num_tokens": 574089.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.4592825189232825,
+      "epoch": 0.312,
+      "grad_norm": 0.9789808392524719,
+      "learning_rate": 9.241132801759692e-07,
+      "loss": 1.2866,
+      "mean_token_accuracy": 0.6407172739505768,
+      "num_tokens": 589811.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.4485271602869034,
+      "epoch": 0.32,
+      "grad_norm": 1.132893681526184,
+      "learning_rate": 9.213637613417652e-07,
+      "loss": 1.3512,
+      "mean_token_accuracy": 0.6424966964870691,
+      "num_tokens": 605471.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.4448492050170898,
+      "epoch": 0.328,
+      "grad_norm": 0.8566483855247498,
+      "learning_rate": 9.186142425075612e-07,
+      "loss": 1.3409,
+      "mean_token_accuracy": 0.6550276458263398,
+      "num_tokens": 620546.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.520164243876934,
+      "epoch": 0.336,
+      "grad_norm": 1.069487452507019,
+      "learning_rate": 9.158647236733571e-07,
+      "loss": 1.3899,
+      "mean_token_accuracy": 0.6334433287382126,
+      "num_tokens": 633464.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.438837230950594,
+      "epoch": 0.344,
+      "grad_norm": 0.9344055652618408,
+      "learning_rate": 9.131152048391531e-07,
+      "loss": 1.2575,
+      "mean_token_accuracy": 0.6545429665595293,
+      "num_tokens": 649071.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.4020009264349937,
+      "epoch": 0.352,
+      "grad_norm": 0.7401624321937561,
+      "learning_rate": 9.103656860049491e-07,
+      "loss": 1.2571,
+      "mean_token_accuracy": 0.6630169056355953,
+      "num_tokens": 665205.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.4204766362905503,
+      "epoch": 0.36,
+      "grad_norm": 0.6837717890739441,
+      "learning_rate": 9.076161671707451e-07,
+      "loss": 1.3233,
+      "mean_token_accuracy": 0.6543396785855293,
+      "num_tokens": 681625.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.5160297207534312,
+      "epoch": 0.368,
+      "grad_norm": 0.963768720626831,
+      "learning_rate": 9.048666483365411e-07,
+      "loss": 1.3115,
+      "mean_token_accuracy": 0.6477277036756277,
+      "num_tokens": 695793.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.4665997616946698,
+      "epoch": 0.376,
+      "grad_norm": 0.8436604738235474,
+      "learning_rate": 9.021171295023371e-07,
+      "loss": 1.2849,
+      "mean_token_accuracy": 0.6575528148561716,
+      "num_tokens": 709941.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.35268774330616,
+      "epoch": 0.384,
+      "grad_norm": 0.5754812955856323,
+      "learning_rate": 8.993676106681331e-07,
+      "loss": 1.1586,
+      "mean_token_accuracy": 0.6813954003155231,
+      "num_tokens": 727938.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.4338405415415765,
+      "epoch": 0.392,
+      "grad_norm": 0.809728741645813,
+      "learning_rate": 8.96618091833929e-07,
+      "loss": 1.2498,
+      "mean_token_accuracy": 0.6674149315804243,
+      "num_tokens": 742288.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.3899291299283505,
+      "epoch": 0.4,
+      "grad_norm": 0.8144301176071167,
+      "learning_rate": 8.93868572999725e-07,
+      "loss": 1.2708,
+      "mean_token_accuracy": 0.6641967866569758,
+      "num_tokens": 756290.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.392310130596161,
+      "epoch": 0.408,
+      "grad_norm": 0.729603111743927,
+      "learning_rate": 8.91119054165521e-07,
+      "loss": 1.2116,
+      "mean_token_accuracy": 0.6670112654566764,
+      "num_tokens": 771641.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.3988941840827465,
+      "epoch": 0.416,
+      "grad_norm": 0.8343493342399597,
+      "learning_rate": 8.88369535331317e-07,
+      "loss": 1.1945,
+      "mean_token_accuracy": 0.675208692997694,
+      "num_tokens": 785255.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.373506922274828,
+      "epoch": 0.424,
+      "grad_norm": 0.5414552092552185,
+      "learning_rate": 8.85620016497113e-07,
+      "loss": 1.2322,
+      "mean_token_accuracy": 0.6692581787705422,
+      "num_tokens": 801256.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.3819667909294366,
+      "epoch": 0.432,
+      "grad_norm": 0.7159795165061951,
+      "learning_rate": 8.82870497662909e-07,
+      "loss": 1.1919,
+      "mean_token_accuracy": 0.6702349409461021,
+      "num_tokens": 816436.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.3742642417550086,
+      "epoch": 0.44,
+      "grad_norm": 0.6191155910491943,
+      "learning_rate": 8.80120978828705e-07,
+      "loss": 1.2179,
+      "mean_token_accuracy": 0.6660036187618971,
+      "num_tokens": 830892.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.4100160527974368,
+      "epoch": 0.448,
+      "grad_norm": 0.6038439273834229,
+      "learning_rate": 8.773714599945009e-07,
+      "loss": 1.2194,
+      "mean_token_accuracy": 0.6648291684687138,
+      "num_tokens": 845413.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.3719368264079095,
+      "epoch": 0.456,
+      "grad_norm": 0.6069778203964233,
+      "learning_rate": 8.746219411602969e-07,
+      "loss": 1.2522,
+      "mean_token_accuracy": 0.6610075645148754,
+      "num_tokens": 860409.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.403072853386402,
+      "epoch": 0.464,
+      "grad_norm": 0.8242809772491455,
+      "learning_rate": 8.71872422326093e-07,
+      "loss": 1.205,
+      "mean_token_accuracy": 0.6765511088073254,
+      "num_tokens": 873795.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.3861247181892395,
+      "epoch": 0.472,
+      "grad_norm": 0.7247884273529053,
+      "learning_rate": 8.691229034918889e-07,
+      "loss": 1.1738,
+      "mean_token_accuracy": 0.6671203374862671,
+      "num_tokens": 887567.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.3157524734735488,
+      "epoch": 0.48,
+      "grad_norm": 1.043849229812622,
+      "learning_rate": 8.663733846576849e-07,
+      "loss": 1.1864,
+      "mean_token_accuracy": 0.67687898427248,
+      "num_tokens": 903500.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.3670225895941257,
+      "epoch": 0.488,
+      "grad_norm": 0.6287969946861267,
+      "learning_rate": 8.636238658234809e-07,
+      "loss": 1.2227,
+      "mean_token_accuracy": 0.6774305209517479,
+      "num_tokens": 917591.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.4043861143290997,
+      "epoch": 0.496,
+      "grad_norm": 0.6480540037155151,
+      "learning_rate": 8.608743469892769e-07,
+      "loss": 1.2478,
+      "mean_token_accuracy": 0.6648495178669691,
+      "num_tokens": 931977.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.2790821947157383,
+      "epoch": 0.504,
+      "grad_norm": 0.6468939781188965,
+      "learning_rate": 8.581248281550728e-07,
+      "loss": 1.1545,
+      "mean_token_accuracy": 0.6829771518707275,
+      "num_tokens": 948897.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.2587309643626212,
+      "epoch": 0.512,
+      "grad_norm": 0.756490170955658,
+      "learning_rate": 8.553753093208688e-07,
+      "loss": 1.0904,
+      "mean_token_accuracy": 0.6901254523545504,
+      "num_tokens": 965001.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.3130459561944008,
+      "epoch": 0.52,
+      "grad_norm": 0.6585144400596619,
+      "learning_rate": 8.526257904866649e-07,
+      "loss": 1.1556,
+      "mean_token_accuracy": 0.6796729028224945,
+      "num_tokens": 981274.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.311295585334301,
+      "epoch": 0.528,
+      "grad_norm": 0.7502710223197937,
+      "learning_rate": 8.498762716524608e-07,
+      "loss": 1.1656,
+      "mean_token_accuracy": 0.6799918249249458,
+      "num_tokens": 997379.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.296420407295227,
+      "epoch": 0.536,
+      "grad_norm": 0.7529589533805847,
+      "learning_rate": 8.471267528182568e-07,
+      "loss": 1.1843,
+      "mean_token_accuracy": 0.6738614052534103,
+      "num_tokens": 1013352.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.3494001857936382,
+      "epoch": 0.544,
+      "grad_norm": 0.5738757252693176,
+      "learning_rate": 8.443772339840528e-07,
+      "loss": 1.144,
+      "mean_token_accuracy": 0.6632784813642502,
+      "num_tokens": 1029139.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.2736155807971954,
+      "epoch": 0.552,
+      "grad_norm": 0.7190117835998535,
+      "learning_rate": 8.416277151498488e-07,
+      "loss": 1.1806,
+      "mean_token_accuracy": 0.6787780053913594,
+      "num_tokens": 1045055.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.273686296120286,
+      "epoch": 0.56,
+      "grad_norm": 0.6301732063293457,
+      "learning_rate": 8.388781963156447e-07,
+      "loss": 1.1266,
+      "mean_token_accuracy": 0.6862449984997511,
+      "num_tokens": 1061665.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.2967881709337234,
+      "epoch": 0.568,
+      "grad_norm": 0.6849706768989563,
+      "learning_rate": 8.361286774814407e-07,
+      "loss": 1.1836,
+      "mean_token_accuracy": 0.675114393979311,
+      "num_tokens": 1076783.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.293698250502348,
+      "epoch": 0.576,
+      "grad_norm": 0.8003822565078735,
+      "learning_rate": 8.333791586472368e-07,
+      "loss": 1.1816,
+      "mean_token_accuracy": 0.6739512585103512,
+      "num_tokens": 1092863.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.2287389039993286,
+      "epoch": 0.584,
+      "grad_norm": 0.6071404218673706,
+      "learning_rate": 8.306296398130327e-07,
+      "loss": 1.1066,
+      "mean_token_accuracy": 0.6892683796584607,
+      "num_tokens": 1109869.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.205421121418476,
+      "epoch": 0.592,
+      "grad_norm": 0.6821198463439941,
+      "learning_rate": 8.278801209788287e-07,
+      "loss": 1.1391,
+      "mean_token_accuracy": 0.6925450332462788,
+      "num_tokens": 1125614.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.2849964290857314,
+      "epoch": 0.6,
+      "grad_norm": 0.5708684325218201,
+      "learning_rate": 8.251306021446246e-07,
+      "loss": 1.1471,
+      "mean_token_accuracy": 0.6788520190864802,
+      "num_tokens": 1140573.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.2744114309549333,
+      "epoch": 0.608,
+      "grad_norm": 0.7616786956787109,
+      "learning_rate": 8.223810833104207e-07,
+      "loss": 1.1777,
+      "mean_token_accuracy": 0.6805129069834948,
+      "num_tokens": 1155477.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.2422254398465156,
+      "epoch": 0.616,
+      "grad_norm": 0.8303386569023132,
+      "learning_rate": 8.196315644762166e-07,
+      "loss": 1.1692,
+      "mean_token_accuracy": 0.6900120198726654,
+      "num_tokens": 1169887.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.2660817682743073,
+      "epoch": 0.624,
+      "grad_norm": 0.5202187895774841,
+      "learning_rate": 8.168820456420126e-07,
+      "loss": 1.1537,
+      "mean_token_accuracy": 0.6705128997564316,
+      "num_tokens": 1186324.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.2406820882111789,
+      "epoch": 0.632,
+      "grad_norm": 0.7239068150520325,
+      "learning_rate": 8.141325268078087e-07,
+      "loss": 1.1585,
+      "mean_token_accuracy": 0.677312508970499,
+      "num_tokens": 1201664.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.2289198808372022,
+      "epoch": 0.64,
+      "grad_norm": 0.5939375162124634,
+      "learning_rate": 8.113830079736046e-07,
+      "loss": 1.1053,
+      "mean_token_accuracy": 0.6843044184148311,
+      "num_tokens": 1217925.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.2725118707865477,
+      "epoch": 0.648,
+      "grad_norm": 0.7037603855133057,
+      "learning_rate": 8.086334891394006e-07,
+      "loss": 1.1437,
+      "mean_token_accuracy": 0.6814427956938743,
+      "num_tokens": 1231061.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.2992851927876472,
+      "epoch": 0.656,
+      "grad_norm": 0.8515540361404419,
+      "learning_rate": 8.058839703051965e-07,
+      "loss": 1.2121,
+      "mean_token_accuracy": 0.6711709387600422,
+      "num_tokens": 1244058.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.2262615151703358,
+      "epoch": 0.664,
+      "grad_norm": 0.6631777286529541,
+      "learning_rate": 8.031344514709926e-07,
+      "loss": 1.1458,
+      "mean_token_accuracy": 0.6818120799958706,
+      "num_tokens": 1259582.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.2693677991628647,
+      "epoch": 0.672,
+      "grad_norm": 0.8915537595748901,
+      "learning_rate": 8.003849326367885e-07,
+      "loss": 1.1633,
+      "mean_token_accuracy": 0.6815830297768116,
+      "num_tokens": 1273831.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.215834778547287,
+      "epoch": 0.68,
+      "grad_norm": 0.5939379930496216,
+      "learning_rate": 7.976354138025845e-07,
+      "loss": 1.0981,
+      "mean_token_accuracy": 0.6865571141242981,
+      "num_tokens": 1289571.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.1748904675245284,
+      "epoch": 0.688,
+      "grad_norm": 0.5998817086219788,
+      "learning_rate": 7.948858949683806e-07,
+      "loss": 1.0692,
+      "mean_token_accuracy": 0.693486251682043,
+      "num_tokens": 1306133.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.2221910648047924,
+      "epoch": 0.696,
+      "grad_norm": 0.748798668384552,
+      "learning_rate": 7.921363761341765e-07,
+      "loss": 1.1412,
+      "mean_token_accuracy": 0.6878058951348066,
+      "num_tokens": 1321464.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.2369954086840154,
+      "epoch": 0.704,
+      "grad_norm": 1.0195012092590332,
+      "learning_rate": 7.893868572999725e-07,
+      "loss": 1.1408,
+      "mean_token_accuracy": 0.681305569410324,
+      "num_tokens": 1335778.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.1726680658757687,
+      "epoch": 0.712,
+      "grad_norm": 0.7860333919525146,
+      "learning_rate": 7.866373384657684e-07,
+      "loss": 1.091,
+      "mean_token_accuracy": 0.6857907593250274,
+      "num_tokens": 1352915.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.223223304003477,
+      "epoch": 0.72,
+      "grad_norm": 0.9237163662910461,
+      "learning_rate": 7.838878196315645e-07,
+      "loss": 1.1373,
+      "mean_token_accuracy": 0.6926517814397812,
+      "num_tokens": 1367612.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.1455084159970284,
+      "epoch": 0.728,
+      "grad_norm": 0.5603041648864746,
+      "learning_rate": 7.811383007973604e-07,
+      "loss": 1.0911,
+      "mean_token_accuracy": 0.6972605399787426,
+      "num_tokens": 1383035.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.1558292523026465,
+      "epoch": 0.736,
+      "grad_norm": 0.7899258136749268,
+      "learning_rate": 7.783887819631564e-07,
+      "loss": 1.1121,
+      "mean_token_accuracy": 0.6864722102880478,
+      "num_tokens": 1399802.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.210861251503229,
+      "epoch": 0.744,
+      "grad_norm": 0.8369595408439636,
+      "learning_rate": 7.756392631289525e-07,
+      "loss": 1.1218,
+      "mean_token_accuracy": 0.6842097207903862,
+      "num_tokens": 1415756.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.2210182018578053,
+      "epoch": 0.752,
+      "grad_norm": 0.710249125957489,
+      "learning_rate": 7.728897442947484e-07,
+      "loss": 1.1491,
+      "mean_token_accuracy": 0.691110260784626,
+      "num_tokens": 1430566.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.2260907642543315,
+      "epoch": 0.76,
+      "grad_norm": 0.6935760974884033,
+      "learning_rate": 7.701402254605444e-07,
+      "loss": 1.1606,
+      "mean_token_accuracy": 0.6785398446023464,
+      "num_tokens": 1446008.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.2416882522404193,
+      "epoch": 0.768,
+      "grad_norm": 0.8145465850830078,
+      "learning_rate": 7.673907066263403e-07,
+      "loss": 1.1903,
+      "mean_token_accuracy": 0.6766292177140713,
+      "num_tokens": 1460630.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2305906921625138,
+      "epoch": 0.776,
+      "grad_norm": 0.6397637724876404,
+      "learning_rate": 7.646411877921364e-07,
+      "loss": 1.1322,
+      "mean_token_accuracy": 0.686901830881834,
+      "num_tokens": 1475286.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.2152808040380478,
+      "epoch": 0.784,
+      "grad_norm": 0.7708243727684021,
+      "learning_rate": 7.618916689579323e-07,
+      "loss": 1.1465,
+      "mean_token_accuracy": 0.6885192066431045,
+      "num_tokens": 1488831.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.2126370012760161,
+      "epoch": 0.792,
+      "grad_norm": 0.7606609463691711,
+      "learning_rate": 7.591421501237283e-07,
+      "loss": 1.1499,
+      "mean_token_accuracy": 0.6837481513619423,
+      "num_tokens": 1502143.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.201651045680046,
+      "epoch": 0.8,
+      "grad_norm": 0.6086705327033997,
+      "learning_rate": 7.563926312895244e-07,
+      "loss": 1.0735,
+      "mean_token_accuracy": 0.7014785043895244,
+      "num_tokens": 1517934.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.2058428205549716,
+      "epoch": 0.808,
+      "grad_norm": 0.49491190910339355,
+      "learning_rate": 7.536431124553203e-07,
+      "loss": 1.1164,
+      "mean_token_accuracy": 0.6956117272377014,
+      "num_tokens": 1533374.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.1543868333101273,
+      "epoch": 0.816,
+      "grad_norm": 0.7286425232887268,
+      "learning_rate": 7.508935936211163e-07,
+      "loss": 1.1223,
+      "mean_token_accuracy": 0.6940035171806812,
+      "num_tokens": 1548879.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.1904160492122173,
+      "epoch": 0.824,
+      "grad_norm": 0.7718061804771423,
+      "learning_rate": 7.481440747869122e-07,
+      "loss": 1.1458,
+      "mean_token_accuracy": 0.6934111632406712,
+      "num_tokens": 1563736.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.1673480425029994,
+      "epoch": 0.832,
+      "grad_norm": 0.6239304542541504,
+      "learning_rate": 7.453945559527083e-07,
+      "loss": 1.0974,
+      "mean_token_accuracy": 0.6918889433145523,
+      "num_tokens": 1580364.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.1715149961411953,
+      "epoch": 0.84,
+      "grad_norm": 0.6284940242767334,
+      "learning_rate": 7.426450371185042e-07,
+      "loss": 1.1331,
+      "mean_token_accuracy": 0.6936986103653908,
+      "num_tokens": 1595277.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.1704499389976264,
+      "epoch": 0.848,
+      "grad_norm": 0.6014586687088013,
+      "learning_rate": 7.398955182843002e-07,
+      "loss": 1.0852,
+      "mean_token_accuracy": 0.6870665144175291,
+      "num_tokens": 1611046.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.1872769869863986,
+      "epoch": 0.856,
+      "grad_norm": 0.683362603187561,
+      "learning_rate": 7.371459994500962e-07,
+      "loss": 1.1458,
+      "mean_token_accuracy": 0.6825452599674463,
+      "num_tokens": 1626105.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.1762015298008919,
+      "epoch": 0.864,
+      "grad_norm": 0.7485630512237549,
+      "learning_rate": 7.343964806158922e-07,
+      "loss": 1.0449,
+      "mean_token_accuracy": 0.6980511479079723,
+      "num_tokens": 1642095.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.2044708047062158,
+      "epoch": 0.872,
+      "grad_norm": 0.7228647470474243,
+      "learning_rate": 7.316469617816882e-07,
+      "loss": 1.117,
+      "mean_token_accuracy": 0.6872781366109848,
+      "num_tokens": 1656064.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.1704509019851685,
+      "epoch": 0.88,
+      "grad_norm": 0.6274661421775818,
+      "learning_rate": 7.288974429474841e-07,
+      "loss": 1.0567,
+      "mean_token_accuracy": 0.6986218981444836,
+      "num_tokens": 1671905.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.1555724613368512,
+      "epoch": 0.888,
+      "grad_norm": 0.5717065930366516,
+      "learning_rate": 7.261479241132802e-07,
+      "loss": 1.0613,
+      "mean_token_accuracy": 0.6985699016600847,
+      "num_tokens": 1687831.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.1825853794813157,
+      "epoch": 0.896,
+      "grad_norm": 0.5806442499160767,
+      "learning_rate": 7.233984052790761e-07,
+      "loss": 1.1196,
+      "mean_token_accuracy": 0.6930595509707927,
+      "num_tokens": 1703242.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.1857046082615852,
+      "epoch": 0.904,
+      "grad_norm": 0.7729993462562561,
+      "learning_rate": 7.206488864448721e-07,
+      "loss": 1.0937,
+      "mean_token_accuracy": 0.692601241916418,
+      "num_tokens": 1717420.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.15954297054559,
+      "epoch": 0.912,
+      "grad_norm": 0.5793495178222656,
+      "learning_rate": 7.178993676106681e-07,
+      "loss": 1.0322,
+      "mean_token_accuracy": 0.7003567360341549,
+      "num_tokens": 1734570.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.1955730587244033,
+      "epoch": 0.92,
+      "grad_norm": 0.6606018543243408,
+      "learning_rate": 7.151498487764641e-07,
+      "loss": 1.1115,
+      "mean_token_accuracy": 0.6888970002532006,
+      "num_tokens": 1749977.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.1169864691793918,
+      "epoch": 0.928,
+      "grad_norm": 0.511619508266449,
+      "learning_rate": 7.124003299422601e-07,
+      "loss": 1.0671,
+      "mean_token_accuracy": 0.7025054290890693,
+      "num_tokens": 1766717.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.145845735212788,
+      "epoch": 0.936,
+      "grad_norm": 0.7826500535011292,
+      "learning_rate": 7.09650811108056e-07,
+      "loss": 1.0499,
+      "mean_token_accuracy": 0.6996002644300461,
+      "num_tokens": 1781248.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.1137450352311133,
+      "epoch": 0.944,
+      "grad_norm": 0.6092593669891357,
+      "learning_rate": 7.069012922738521e-07,
+      "loss": 1.0631,
+      "mean_token_accuracy": 0.7020095460116863,
+      "num_tokens": 1797901.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.2010398358106613,
+      "epoch": 0.952,
+      "grad_norm": 0.9775738716125488,
+      "learning_rate": 7.04151773439648e-07,
+      "loss": 1.1548,
+      "mean_token_accuracy": 0.6882500603795052,
+      "num_tokens": 1811944.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.1422110214829444,
+      "epoch": 0.96,
+      "grad_norm": 0.715650737285614,
+      "learning_rate": 7.01402254605444e-07,
+      "loss": 1.1148,
+      "mean_token_accuracy": 0.6927286379039288,
+      "num_tokens": 1828178.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.14734987616539,
+      "epoch": 0.968,
+      "grad_norm": 0.5900126099586487,
+      "learning_rate": 6.9865273577124e-07,
+      "loss": 1.0809,
+      "mean_token_accuracy": 0.6929037302732468,
+      "num_tokens": 1845334.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.1666462287306785,
+      "epoch": 0.976,
+      "grad_norm": 0.7052849531173706,
+      "learning_rate": 6.95903216937036e-07,
+      "loss": 1.086,
+      "mean_token_accuracy": 0.6887565299868583,
+      "num_tokens": 1859542.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.127211359143257,
+      "epoch": 0.984,
+      "grad_norm": 0.9729690551757812,
+      "learning_rate": 6.93153698102832e-07,
+      "loss": 1.0588,
+      "mean_token_accuracy": 0.69860158264637,
+      "num_tokens": 1876209.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.1535819880664349,
+      "epoch": 0.992,
+      "grad_norm": 0.6623355746269226,
+      "learning_rate": 6.904041792686279e-07,
+      "loss": 1.0543,
+      "mean_token_accuracy": 0.695023113116622,
+      "num_tokens": 1893007.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.1296846129000186,
+      "epoch": 1.0,
+      "grad_norm": 0.6806105375289917,
+      "learning_rate": 6.87654660434424e-07,
+      "loss": 1.089,
+      "mean_token_accuracy": 0.6975006379187108,
+      "num_tokens": 1908628.0,
+      "step": 1250
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3750,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.74969936717742e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

full-lora/checkpoint-1250/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:373064d186b379b705f3701bc2192b8d633e746e079c3d5e972608780158085a
+size 6417

full-lora/checkpoint-2500/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-2-7b-chat-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-2-7b-chat-hf
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

full-lora/checkpoint-2500/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

full-lora/checkpoint-2500/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea490ee39d7c12580c65616f062eea4356694707e3e1b0acfbce1883c0de0908
+size 639691872

full-lora/checkpoint-2500/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\n' + system_message + '\n<</SYS>>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}

full-lora/checkpoint-2500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50511cce1729154ff4ed89de392872e886bd3b628988876931b9e054d1e20340
+size 1279641507

full-lora/checkpoint-2500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6338797a9f196294e9ad75c2a3b6e746b32ba543fe0059650074c0e5cb8bb40f
+size 14645

full-lora/checkpoint-2500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ad402eb28146d764966647b6826632ce0185367514a12153b690ebf7f1f43b
+size 1465

full-lora/checkpoint-2500/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

full-lora/checkpoint-2500/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

full-lora/checkpoint-2500/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

full-lora/checkpoint-2500/trainer_state.json

@@ -0,0 +1,2534 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.9034420549869537,
+      "epoch": 0.008,
+      "grad_norm": 1.8304097652435303,
+      "learning_rate": 7.964601769911503e-08,
+      "loss": 1.9907,
+      "mean_token_accuracy": 0.6042250197380781,
+      "num_tokens": 15409.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.9475356195122003,
+      "epoch": 0.016,
+      "grad_norm": 1.5987296104431152,
+      "learning_rate": 1.68141592920354e-07,
+      "loss": 2.0397,
+      "mean_token_accuracy": 0.5890234299004078,
+      "num_tokens": 30650.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.9629276197403669,
+      "epoch": 0.024,
+      "grad_norm": 2.1185450553894043,
+      "learning_rate": 2.5663716814159294e-07,
+      "loss": 2.0312,
+      "mean_token_accuracy": 0.5870847202837467,
+      "num_tokens": 45561.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.9790993589907885,
+      "epoch": 0.032,
+      "grad_norm": 2.317858934402466,
+      "learning_rate": 3.451327433628318e-07,
+      "loss": 2.0456,
+      "mean_token_accuracy": 0.5804481774568557,
+      "num_tokens": 59568.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.922547810524702,
+      "epoch": 0.04,
+      "grad_norm": 1.5205438137054443,
+      "learning_rate": 4.3362831858407076e-07,
+      "loss": 1.9502,
+      "mean_token_accuracy": 0.5938452776521445,
+      "num_tokens": 75405.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.03425248041749,
+      "epoch": 0.048,
+      "grad_norm": 2.2233166694641113,
+      "learning_rate": 5.221238938053097e-07,
+      "loss": 2.1455,
+      "mean_token_accuracy": 0.5820784538984298,
+      "num_tokens": 88641.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.9562045603990554,
+      "epoch": 0.056,
+      "grad_norm": 1.6535303592681885,
+      "learning_rate": 6.106194690265486e-07,
+      "loss": 1.9628,
+      "mean_token_accuracy": 0.5903334401547908,
+      "num_tokens": 104448.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.9437448482960462,
+      "epoch": 0.064,
+      "grad_norm": 2.1216635704040527,
+      "learning_rate": 6.991150442477876e-07,
+      "loss": 2.0221,
+      "mean_token_accuracy": 0.5954494498670101,
+      "num_tokens": 119162.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.9662446774542331,
+      "epoch": 0.072,
+      "grad_norm": 1.6397205591201782,
+      "learning_rate": 7.876106194690266e-07,
+      "loss": 1.8199,
+      "mean_token_accuracy": 0.6050666151568294,
+      "num_tokens": 136337.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.9452362310141325,
+      "epoch": 0.08,
+      "grad_norm": 2.1384687423706055,
+      "learning_rate": 8.761061946902655e-07,
+      "loss": 2.0152,
+      "mean_token_accuracy": 0.6010695319622755,
+      "num_tokens": 150710.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.077597926557064,
+      "epoch": 0.088,
+      "grad_norm": 2.1343936920166016,
+      "learning_rate": 9.646017699115042e-07,
+      "loss": 2.0805,
+      "mean_token_accuracy": 0.5725633725523949,
+      "num_tokens": 163196.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.0407875191420317,
+      "epoch": 0.096,
+      "grad_norm": 2.0486209392547607,
+      "learning_rate": 9.983502886994776e-07,
+      "loss": 1.9709,
+      "mean_token_accuracy": 0.5931926343590022,
+      "num_tokens": 177368.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.0551968751475216,
+      "epoch": 0.104,
+      "grad_norm": 1.6275029182434082,
+      "learning_rate": 9.956007698652736e-07,
+      "loss": 1.8687,
+      "mean_token_accuracy": 0.6001733940094709,
+      "num_tokens": 191720.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.0219032436609268,
+      "epoch": 0.112,
+      "grad_norm": 1.6011073589324951,
+      "learning_rate": 9.928512510310695e-07,
+      "loss": 1.7826,
+      "mean_token_accuracy": 0.6063280884176493,
+      "num_tokens": 207354.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.057712158560753,
+      "epoch": 0.12,
+      "grad_norm": 1.6710939407348633,
+      "learning_rate": 9.901017321968655e-07,
+      "loss": 1.6872,
+      "mean_token_accuracy": 0.6104031000286341,
+      "num_tokens": 223850.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.1023272581398487,
+      "epoch": 0.128,
+      "grad_norm": 1.8599427938461304,
+      "learning_rate": 9.873522133626614e-07,
+      "loss": 1.6716,
+      "mean_token_accuracy": 0.6082586232572794,
+      "num_tokens": 239768.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1792589366436004,
+      "epoch": 0.136,
+      "grad_norm": 1.7112274169921875,
+      "learning_rate": 9.846026945284574e-07,
+      "loss": 1.7211,
+      "mean_token_accuracy": 0.606986602395773,
+      "num_tokens": 254892.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.2239510141313077,
+      "epoch": 0.144,
+      "grad_norm": 1.8953965902328491,
+      "learning_rate": 9.818531756942533e-07,
+      "loss": 1.7478,
+      "mean_token_accuracy": 0.6006653252989054,
+      "num_tokens": 269298.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1604003377258778,
+      "epoch": 0.152,
+      "grad_norm": 1.2709853649139404,
+      "learning_rate": 9.791036568600495e-07,
+      "loss": 1.5892,
+      "mean_token_accuracy": 0.6230274979025126,
+      "num_tokens": 283711.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1698237597942351,
+      "epoch": 0.16,
+      "grad_norm": 1.2409337759017944,
+      "learning_rate": 9.763541380258455e-07,
+      "loss": 1.4959,
+      "mean_token_accuracy": 0.6206645734608174,
+      "num_tokens": 301123.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.3337292484939098,
+      "epoch": 0.168,
+      "grad_norm": 1.0640740394592285,
+      "learning_rate": 9.736046191916414e-07,
+      "loss": 1.6387,
+      "mean_token_accuracy": 0.6070953067392111,
+      "num_tokens": 315342.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.2350904777646066,
+      "epoch": 0.176,
+      "grad_norm": 1.3057981729507446,
+      "learning_rate": 9.708551003574374e-07,
+      "loss": 1.5271,
+      "mean_token_accuracy": 0.6268062952905893,
+      "num_tokens": 332035.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.2457694321870805,
+      "epoch": 0.184,
+      "grad_norm": 1.434810996055603,
+      "learning_rate": 9.681055815232333e-07,
+      "loss": 1.4719,
+      "mean_token_accuracy": 0.627309649437666,
+      "num_tokens": 348533.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.3725072897970676,
+      "epoch": 0.192,
+      "grad_norm": 1.203586459159851,
+      "learning_rate": 9.653560626890293e-07,
+      "loss": 1.4614,
+      "mean_token_accuracy": 0.618980173021555,
+      "num_tokens": 363915.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.3240537498146296,
+      "epoch": 0.2,
+      "grad_norm": 1.5373510122299194,
+      "learning_rate": 9.626065438548252e-07,
+      "loss": 1.4521,
+      "mean_token_accuracy": 0.6376201532781124,
+      "num_tokens": 380021.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.4129172183573246,
+      "epoch": 0.208,
+      "grad_norm": 1.0626189708709717,
+      "learning_rate": 9.598570250206214e-07,
+      "loss": 1.4791,
+      "mean_token_accuracy": 0.639534817636013,
+      "num_tokens": 394415.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.4732031315565108,
+      "epoch": 0.216,
+      "grad_norm": 1.2482506036758423,
+      "learning_rate": 9.571075061864174e-07,
+      "loss": 1.5304,
+      "mean_token_accuracy": 0.6208007004112005,
+      "num_tokens": 408643.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.4145490050315856,
+      "epoch": 0.224,
+      "grad_norm": 1.069368600845337,
+      "learning_rate": 9.543579873522133e-07,
+      "loss": 1.3775,
+      "mean_token_accuracy": 0.6412477158010006,
+      "num_tokens": 423952.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.4205268509685993,
+      "epoch": 0.232,
+      "grad_norm": 0.8400747776031494,
+      "learning_rate": 9.516084685180094e-07,
+      "loss": 1.4327,
+      "mean_token_accuracy": 0.6374875742942094,
+      "num_tokens": 438452.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.4074747256934643,
+      "epoch": 0.24,
+      "grad_norm": 1.1713480949401855,
+      "learning_rate": 9.488589496838053e-07,
+      "loss": 1.4076,
+      "mean_token_accuracy": 0.640987902879715,
+      "num_tokens": 453417.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.4747156761586666,
+      "epoch": 0.248,
+      "grad_norm": 0.9282711744308472,
+      "learning_rate": 9.461094308496013e-07,
+      "loss": 1.4397,
+      "mean_token_accuracy": 0.6315288979560136,
+      "num_tokens": 467659.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.407252886891365,
+      "epoch": 0.256,
+      "grad_norm": 0.6602013111114502,
+      "learning_rate": 9.433599120153973e-07,
+      "loss": 1.3759,
+      "mean_token_accuracy": 0.6466614466160536,
+      "num_tokens": 484442.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.4545464389026166,
+      "epoch": 0.264,
+      "grad_norm": 1.206853985786438,
+      "learning_rate": 9.406103931811933e-07,
+      "loss": 1.4166,
+      "mean_token_accuracy": 0.623085155710578,
+      "num_tokens": 498395.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.4992636159062385,
+      "epoch": 0.272,
+      "grad_norm": 0.8671019673347473,
+      "learning_rate": 9.378608743469893e-07,
+      "loss": 1.3696,
+      "mean_token_accuracy": 0.6376510679721832,
+      "num_tokens": 513410.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.4165021255612373,
+      "epoch": 0.28,
+      "grad_norm": 0.7934819459915161,
+      "learning_rate": 9.351113555127852e-07,
+      "loss": 1.2903,
+      "mean_token_accuracy": 0.6567342653870583,
+      "num_tokens": 529739.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.5230652637779714,
+      "epoch": 0.288,
+      "grad_norm": 0.7465261220932007,
+      "learning_rate": 9.323618366785813e-07,
+      "loss": 1.369,
+      "mean_token_accuracy": 0.6413242511451245,
+      "num_tokens": 543258.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.4946719333529472,
+      "epoch": 0.296,
+      "grad_norm": 0.7156000733375549,
+      "learning_rate": 9.296123178443772e-07,
+      "loss": 1.3848,
+      "mean_token_accuracy": 0.6313502866774797,
+      "num_tokens": 558401.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.433413552492857,
+      "epoch": 0.304,
+      "grad_norm": 0.8262195587158203,
+      "learning_rate": 9.268627990101732e-07,
+      "loss": 1.346,
+      "mean_token_accuracy": 0.6439653303474188,
+      "num_tokens": 574089.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.4592825189232825,
+      "epoch": 0.312,
+      "grad_norm": 0.9789808392524719,
+      "learning_rate": 9.241132801759692e-07,
+      "loss": 1.2866,
+      "mean_token_accuracy": 0.6407172739505768,
+      "num_tokens": 589811.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.4485271602869034,
+      "epoch": 0.32,
+      "grad_norm": 1.132893681526184,
+      "learning_rate": 9.213637613417652e-07,
+      "loss": 1.3512,
+      "mean_token_accuracy": 0.6424966964870691,
+      "num_tokens": 605471.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.4448492050170898,
+      "epoch": 0.328,
+      "grad_norm": 0.8566483855247498,
+      "learning_rate": 9.186142425075612e-07,
+      "loss": 1.3409,
+      "mean_token_accuracy": 0.6550276458263398,
+      "num_tokens": 620546.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.520164243876934,
+      "epoch": 0.336,
+      "grad_norm": 1.069487452507019,
+      "learning_rate": 9.158647236733571e-07,
+      "loss": 1.3899,
+      "mean_token_accuracy": 0.6334433287382126,
+      "num_tokens": 633464.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.438837230950594,
+      "epoch": 0.344,
+      "grad_norm": 0.9344055652618408,
+      "learning_rate": 9.131152048391531e-07,
+      "loss": 1.2575,
+      "mean_token_accuracy": 0.6545429665595293,
+      "num_tokens": 649071.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.4020009264349937,
+      "epoch": 0.352,
+      "grad_norm": 0.7401624321937561,
+      "learning_rate": 9.103656860049491e-07,
+      "loss": 1.2571,
+      "mean_token_accuracy": 0.6630169056355953,
+      "num_tokens": 665205.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.4204766362905503,
+      "epoch": 0.36,
+      "grad_norm": 0.6837717890739441,
+      "learning_rate": 9.076161671707451e-07,
+      "loss": 1.3233,
+      "mean_token_accuracy": 0.6543396785855293,
+      "num_tokens": 681625.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.5160297207534312,
+      "epoch": 0.368,
+      "grad_norm": 0.963768720626831,
+      "learning_rate": 9.048666483365411e-07,
+      "loss": 1.3115,
+      "mean_token_accuracy": 0.6477277036756277,
+      "num_tokens": 695793.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.4665997616946698,
+      "epoch": 0.376,
+      "grad_norm": 0.8436604738235474,
+      "learning_rate": 9.021171295023371e-07,
+      "loss": 1.2849,
+      "mean_token_accuracy": 0.6575528148561716,
+      "num_tokens": 709941.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.35268774330616,
+      "epoch": 0.384,
+      "grad_norm": 0.5754812955856323,
+      "learning_rate": 8.993676106681331e-07,
+      "loss": 1.1586,
+      "mean_token_accuracy": 0.6813954003155231,
+      "num_tokens": 727938.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.4338405415415765,
+      "epoch": 0.392,
+      "grad_norm": 0.809728741645813,
+      "learning_rate": 8.96618091833929e-07,
+      "loss": 1.2498,
+      "mean_token_accuracy": 0.6674149315804243,
+      "num_tokens": 742288.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.3899291299283505,
+      "epoch": 0.4,
+      "grad_norm": 0.8144301176071167,
+      "learning_rate": 8.93868572999725e-07,
+      "loss": 1.2708,
+      "mean_token_accuracy": 0.6641967866569758,
+      "num_tokens": 756290.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.392310130596161,
+      "epoch": 0.408,
+      "grad_norm": 0.729603111743927,
+      "learning_rate": 8.91119054165521e-07,
+      "loss": 1.2116,
+      "mean_token_accuracy": 0.6670112654566764,
+      "num_tokens": 771641.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.3988941840827465,
+      "epoch": 0.416,
+      "grad_norm": 0.8343493342399597,
+      "learning_rate": 8.88369535331317e-07,
+      "loss": 1.1945,
+      "mean_token_accuracy": 0.675208692997694,
+      "num_tokens": 785255.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.373506922274828,
+      "epoch": 0.424,
+      "grad_norm": 0.5414552092552185,
+      "learning_rate": 8.85620016497113e-07,
+      "loss": 1.2322,
+      "mean_token_accuracy": 0.6692581787705422,
+      "num_tokens": 801256.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.3819667909294366,
+      "epoch": 0.432,
+      "grad_norm": 0.7159795165061951,
+      "learning_rate": 8.82870497662909e-07,
+      "loss": 1.1919,
+      "mean_token_accuracy": 0.6702349409461021,
+      "num_tokens": 816436.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.3742642417550086,
+      "epoch": 0.44,
+      "grad_norm": 0.6191155910491943,
+      "learning_rate": 8.80120978828705e-07,
+      "loss": 1.2179,
+      "mean_token_accuracy": 0.6660036187618971,
+      "num_tokens": 830892.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.4100160527974368,
+      "epoch": 0.448,
+      "grad_norm": 0.6038439273834229,
+      "learning_rate": 8.773714599945009e-07,
+      "loss": 1.2194,
+      "mean_token_accuracy": 0.6648291684687138,
+      "num_tokens": 845413.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.3719368264079095,
+      "epoch": 0.456,
+      "grad_norm": 0.6069778203964233,
+      "learning_rate": 8.746219411602969e-07,
+      "loss": 1.2522,
+      "mean_token_accuracy": 0.6610075645148754,
+      "num_tokens": 860409.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.403072853386402,
+      "epoch": 0.464,
+      "grad_norm": 0.8242809772491455,
+      "learning_rate": 8.71872422326093e-07,
+      "loss": 1.205,
+      "mean_token_accuracy": 0.6765511088073254,
+      "num_tokens": 873795.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.3861247181892395,
+      "epoch": 0.472,
+      "grad_norm": 0.7247884273529053,
+      "learning_rate": 8.691229034918889e-07,
+      "loss": 1.1738,
+      "mean_token_accuracy": 0.6671203374862671,
+      "num_tokens": 887567.0,
+      "step": 590
+    },
+    {
+      "entropy": 1.3157524734735488,
+      "epoch": 0.48,
+      "grad_norm": 1.043849229812622,
+      "learning_rate": 8.663733846576849e-07,
+      "loss": 1.1864,
+      "mean_token_accuracy": 0.67687898427248,
+      "num_tokens": 903500.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.3670225895941257,
+      "epoch": 0.488,
+      "grad_norm": 0.6287969946861267,
+      "learning_rate": 8.636238658234809e-07,
+      "loss": 1.2227,
+      "mean_token_accuracy": 0.6774305209517479,
+      "num_tokens": 917591.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.4043861143290997,
+      "epoch": 0.496,
+      "grad_norm": 0.6480540037155151,
+      "learning_rate": 8.608743469892769e-07,
+      "loss": 1.2478,
+      "mean_token_accuracy": 0.6648495178669691,
+      "num_tokens": 931977.0,
+      "step": 620
+    },
+    {
+      "entropy": 1.2790821947157383,
+      "epoch": 0.504,
+      "grad_norm": 0.6468939781188965,
+      "learning_rate": 8.581248281550728e-07,
+      "loss": 1.1545,
+      "mean_token_accuracy": 0.6829771518707275,
+      "num_tokens": 948897.0,
+      "step": 630
+    },
+    {
+      "entropy": 1.2587309643626212,
+      "epoch": 0.512,
+      "grad_norm": 0.756490170955658,
+      "learning_rate": 8.553753093208688e-07,
+      "loss": 1.0904,
+      "mean_token_accuracy": 0.6901254523545504,
+      "num_tokens": 965001.0,
+      "step": 640
+    },
+    {
+      "entropy": 1.3130459561944008,
+      "epoch": 0.52,
+      "grad_norm": 0.6585144400596619,
+      "learning_rate": 8.526257904866649e-07,
+      "loss": 1.1556,
+      "mean_token_accuracy": 0.6796729028224945,
+      "num_tokens": 981274.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.311295585334301,
+      "epoch": 0.528,
+      "grad_norm": 0.7502710223197937,
+      "learning_rate": 8.498762716524608e-07,
+      "loss": 1.1656,
+      "mean_token_accuracy": 0.6799918249249458,
+      "num_tokens": 997379.0,
+      "step": 660
+    },
+    {
+      "entropy": 1.296420407295227,
+      "epoch": 0.536,
+      "grad_norm": 0.7529589533805847,
+      "learning_rate": 8.471267528182568e-07,
+      "loss": 1.1843,
+      "mean_token_accuracy": 0.6738614052534103,
+      "num_tokens": 1013352.0,
+      "step": 670
+    },
+    {
+      "entropy": 1.3494001857936382,
+      "epoch": 0.544,
+      "grad_norm": 0.5738757252693176,
+      "learning_rate": 8.443772339840528e-07,
+      "loss": 1.144,
+      "mean_token_accuracy": 0.6632784813642502,
+      "num_tokens": 1029139.0,
+      "step": 680
+    },
+    {
+      "entropy": 1.2736155807971954,
+      "epoch": 0.552,
+      "grad_norm": 0.7190117835998535,
+      "learning_rate": 8.416277151498488e-07,
+      "loss": 1.1806,
+      "mean_token_accuracy": 0.6787780053913594,
+      "num_tokens": 1045055.0,
+      "step": 690
+    },
+    {
+      "entropy": 1.273686296120286,
+      "epoch": 0.56,
+      "grad_norm": 0.6301732063293457,
+      "learning_rate": 8.388781963156447e-07,
+      "loss": 1.1266,
+      "mean_token_accuracy": 0.6862449984997511,
+      "num_tokens": 1061665.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.2967881709337234,
+      "epoch": 0.568,
+      "grad_norm": 0.6849706768989563,
+      "learning_rate": 8.361286774814407e-07,
+      "loss": 1.1836,
+      "mean_token_accuracy": 0.675114393979311,
+      "num_tokens": 1076783.0,
+      "step": 710
+    },
+    {
+      "entropy": 1.293698250502348,
+      "epoch": 0.576,
+      "grad_norm": 0.8003822565078735,
+      "learning_rate": 8.333791586472368e-07,
+      "loss": 1.1816,
+      "mean_token_accuracy": 0.6739512585103512,
+      "num_tokens": 1092863.0,
+      "step": 720
+    },
+    {
+      "entropy": 1.2287389039993286,
+      "epoch": 0.584,
+      "grad_norm": 0.6071404218673706,
+      "learning_rate": 8.306296398130327e-07,
+      "loss": 1.1066,
+      "mean_token_accuracy": 0.6892683796584607,
+      "num_tokens": 1109869.0,
+      "step": 730
+    },
+    {
+      "entropy": 1.205421121418476,
+      "epoch": 0.592,
+      "grad_norm": 0.6821198463439941,
+      "learning_rate": 8.278801209788287e-07,
+      "loss": 1.1391,
+      "mean_token_accuracy": 0.6925450332462788,
+      "num_tokens": 1125614.0,
+      "step": 740
+    },
+    {
+      "entropy": 1.2849964290857314,
+      "epoch": 0.6,
+      "grad_norm": 0.5708684325218201,
+      "learning_rate": 8.251306021446246e-07,
+      "loss": 1.1471,
+      "mean_token_accuracy": 0.6788520190864802,
+      "num_tokens": 1140573.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.2744114309549333,
+      "epoch": 0.608,
+      "grad_norm": 0.7616786956787109,
+      "learning_rate": 8.223810833104207e-07,
+      "loss": 1.1777,
+      "mean_token_accuracy": 0.6805129069834948,
+      "num_tokens": 1155477.0,
+      "step": 760
+    },
+    {
+      "entropy": 1.2422254398465156,
+      "epoch": 0.616,
+      "grad_norm": 0.8303386569023132,
+      "learning_rate": 8.196315644762166e-07,
+      "loss": 1.1692,
+      "mean_token_accuracy": 0.6900120198726654,
+      "num_tokens": 1169887.0,
+      "step": 770
+    },
+    {
+      "entropy": 1.2660817682743073,
+      "epoch": 0.624,
+      "grad_norm": 0.5202187895774841,
+      "learning_rate": 8.168820456420126e-07,
+      "loss": 1.1537,
+      "mean_token_accuracy": 0.6705128997564316,
+      "num_tokens": 1186324.0,
+      "step": 780
+    },
+    {
+      "entropy": 1.2406820882111789,
+      "epoch": 0.632,
+      "grad_norm": 0.7239068150520325,
+      "learning_rate": 8.141325268078087e-07,
+      "loss": 1.1585,
+      "mean_token_accuracy": 0.677312508970499,
+      "num_tokens": 1201664.0,
+      "step": 790
+    },
+    {
+      "entropy": 1.2289198808372022,
+      "epoch": 0.64,
+      "grad_norm": 0.5939375162124634,
+      "learning_rate": 8.113830079736046e-07,
+      "loss": 1.1053,
+      "mean_token_accuracy": 0.6843044184148311,
+      "num_tokens": 1217925.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.2725118707865477,
+      "epoch": 0.648,
+      "grad_norm": 0.7037603855133057,
+      "learning_rate": 8.086334891394006e-07,
+      "loss": 1.1437,
+      "mean_token_accuracy": 0.6814427956938743,
+      "num_tokens": 1231061.0,
+      "step": 810
+    },
+    {
+      "entropy": 1.2992851927876472,
+      "epoch": 0.656,
+      "grad_norm": 0.8515540361404419,
+      "learning_rate": 8.058839703051965e-07,
+      "loss": 1.2121,
+      "mean_token_accuracy": 0.6711709387600422,
+      "num_tokens": 1244058.0,
+      "step": 820
+    },
+    {
+      "entropy": 1.2262615151703358,
+      "epoch": 0.664,
+      "grad_norm": 0.6631777286529541,
+      "learning_rate": 8.031344514709926e-07,
+      "loss": 1.1458,
+      "mean_token_accuracy": 0.6818120799958706,
+      "num_tokens": 1259582.0,
+      "step": 830
+    },
+    {
+      "entropy": 1.2693677991628647,
+      "epoch": 0.672,
+      "grad_norm": 0.8915537595748901,
+      "learning_rate": 8.003849326367885e-07,
+      "loss": 1.1633,
+      "mean_token_accuracy": 0.6815830297768116,
+      "num_tokens": 1273831.0,
+      "step": 840
+    },
+    {
+      "entropy": 1.215834778547287,
+      "epoch": 0.68,
+      "grad_norm": 0.5939379930496216,
+      "learning_rate": 7.976354138025845e-07,
+      "loss": 1.0981,
+      "mean_token_accuracy": 0.6865571141242981,
+      "num_tokens": 1289571.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.1748904675245284,
+      "epoch": 0.688,
+      "grad_norm": 0.5998817086219788,
+      "learning_rate": 7.948858949683806e-07,
+      "loss": 1.0692,
+      "mean_token_accuracy": 0.693486251682043,
+      "num_tokens": 1306133.0,
+      "step": 860
+    },
+    {
+      "entropy": 1.2221910648047924,
+      "epoch": 0.696,
+      "grad_norm": 0.748798668384552,
+      "learning_rate": 7.921363761341765e-07,
+      "loss": 1.1412,
+      "mean_token_accuracy": 0.6878058951348066,
+      "num_tokens": 1321464.0,
+      "step": 870
+    },
+    {
+      "entropy": 1.2369954086840154,
+      "epoch": 0.704,
+      "grad_norm": 1.0195012092590332,
+      "learning_rate": 7.893868572999725e-07,
+      "loss": 1.1408,
+      "mean_token_accuracy": 0.681305569410324,
+      "num_tokens": 1335778.0,
+      "step": 880
+    },
+    {
+      "entropy": 1.1726680658757687,
+      "epoch": 0.712,
+      "grad_norm": 0.7860333919525146,
+      "learning_rate": 7.866373384657684e-07,
+      "loss": 1.091,
+      "mean_token_accuracy": 0.6857907593250274,
+      "num_tokens": 1352915.0,
+      "step": 890
+    },
+    {
+      "entropy": 1.223223304003477,
+      "epoch": 0.72,
+      "grad_norm": 0.9237163662910461,
+      "learning_rate": 7.838878196315645e-07,
+      "loss": 1.1373,
+      "mean_token_accuracy": 0.6926517814397812,
+      "num_tokens": 1367612.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.1455084159970284,
+      "epoch": 0.728,
+      "grad_norm": 0.5603041648864746,
+      "learning_rate": 7.811383007973604e-07,
+      "loss": 1.0911,
+      "mean_token_accuracy": 0.6972605399787426,
+      "num_tokens": 1383035.0,
+      "step": 910
+    },
+    {
+      "entropy": 1.1558292523026465,
+      "epoch": 0.736,
+      "grad_norm": 0.7899258136749268,
+      "learning_rate": 7.783887819631564e-07,
+      "loss": 1.1121,
+      "mean_token_accuracy": 0.6864722102880478,
+      "num_tokens": 1399802.0,
+      "step": 920
+    },
+    {
+      "entropy": 1.210861251503229,
+      "epoch": 0.744,
+      "grad_norm": 0.8369595408439636,
+      "learning_rate": 7.756392631289525e-07,
+      "loss": 1.1218,
+      "mean_token_accuracy": 0.6842097207903862,
+      "num_tokens": 1415756.0,
+      "step": 930
+    },
+    {
+      "entropy": 1.2210182018578053,
+      "epoch": 0.752,
+      "grad_norm": 0.710249125957489,
+      "learning_rate": 7.728897442947484e-07,
+      "loss": 1.1491,
+      "mean_token_accuracy": 0.691110260784626,
+      "num_tokens": 1430566.0,
+      "step": 940
+    },
+    {
+      "entropy": 1.2260907642543315,
+      "epoch": 0.76,
+      "grad_norm": 0.6935760974884033,
+      "learning_rate": 7.701402254605444e-07,
+      "loss": 1.1606,
+      "mean_token_accuracy": 0.6785398446023464,
+      "num_tokens": 1446008.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.2416882522404193,
+      "epoch": 0.768,
+      "grad_norm": 0.8145465850830078,
+      "learning_rate": 7.673907066263403e-07,
+      "loss": 1.1903,
+      "mean_token_accuracy": 0.6766292177140713,
+      "num_tokens": 1460630.0,
+      "step": 960
+    },
+    {
+      "entropy": 1.2305906921625138,
+      "epoch": 0.776,
+      "grad_norm": 0.6397637724876404,
+      "learning_rate": 7.646411877921364e-07,
+      "loss": 1.1322,
+      "mean_token_accuracy": 0.686901830881834,
+      "num_tokens": 1475286.0,
+      "step": 970
+    },
+    {
+      "entropy": 1.2152808040380478,
+      "epoch": 0.784,
+      "grad_norm": 0.7708243727684021,
+      "learning_rate": 7.618916689579323e-07,
+      "loss": 1.1465,
+      "mean_token_accuracy": 0.6885192066431045,
+      "num_tokens": 1488831.0,
+      "step": 980
+    },
+    {
+      "entropy": 1.2126370012760161,
+      "epoch": 0.792,
+      "grad_norm": 0.7606609463691711,
+      "learning_rate": 7.591421501237283e-07,
+      "loss": 1.1499,
+      "mean_token_accuracy": 0.6837481513619423,
+      "num_tokens": 1502143.0,
+      "step": 990
+    },
+    {
+      "entropy": 1.201651045680046,
+      "epoch": 0.8,
+      "grad_norm": 0.6086705327033997,
+      "learning_rate": 7.563926312895244e-07,
+      "loss": 1.0735,
+      "mean_token_accuracy": 0.7014785043895244,
+      "num_tokens": 1517934.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.2058428205549716,
+      "epoch": 0.808,
+      "grad_norm": 0.49491190910339355,
+      "learning_rate": 7.536431124553203e-07,
+      "loss": 1.1164,
+      "mean_token_accuracy": 0.6956117272377014,
+      "num_tokens": 1533374.0,
+      "step": 1010
+    },
+    {
+      "entropy": 1.1543868333101273,
+      "epoch": 0.816,
+      "grad_norm": 0.7286425232887268,
+      "learning_rate": 7.508935936211163e-07,
+      "loss": 1.1223,
+      "mean_token_accuracy": 0.6940035171806812,
+      "num_tokens": 1548879.0,
+      "step": 1020
+    },
+    {
+      "entropy": 1.1904160492122173,
+      "epoch": 0.824,
+      "grad_norm": 0.7718061804771423,
+      "learning_rate": 7.481440747869122e-07,
+      "loss": 1.1458,
+      "mean_token_accuracy": 0.6934111632406712,
+      "num_tokens": 1563736.0,
+      "step": 1030
+    },
+    {
+      "entropy": 1.1673480425029994,
+      "epoch": 0.832,
+      "grad_norm": 0.6239304542541504,
+      "learning_rate": 7.453945559527083e-07,
+      "loss": 1.0974,
+      "mean_token_accuracy": 0.6918889433145523,
+      "num_tokens": 1580364.0,
+      "step": 1040
+    },
+    {
+      "entropy": 1.1715149961411953,
+      "epoch": 0.84,
+      "grad_norm": 0.6284940242767334,
+      "learning_rate": 7.426450371185042e-07,
+      "loss": 1.1331,
+      "mean_token_accuracy": 0.6936986103653908,
+      "num_tokens": 1595277.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.1704499389976264,
+      "epoch": 0.848,
+      "grad_norm": 0.6014586687088013,
+      "learning_rate": 7.398955182843002e-07,
+      "loss": 1.0852,
+      "mean_token_accuracy": 0.6870665144175291,
+      "num_tokens": 1611046.0,
+      "step": 1060
+    },
+    {
+      "entropy": 1.1872769869863986,
+      "epoch": 0.856,
+      "grad_norm": 0.683362603187561,
+      "learning_rate": 7.371459994500962e-07,
+      "loss": 1.1458,
+      "mean_token_accuracy": 0.6825452599674463,
+      "num_tokens": 1626105.0,
+      "step": 1070
+    },
+    {
+      "entropy": 1.1762015298008919,
+      "epoch": 0.864,
+      "grad_norm": 0.7485630512237549,
+      "learning_rate": 7.343964806158922e-07,
+      "loss": 1.0449,
+      "mean_token_accuracy": 0.6980511479079723,
+      "num_tokens": 1642095.0,
+      "step": 1080
+    },
+    {
+      "entropy": 1.2044708047062158,
+      "epoch": 0.872,
+      "grad_norm": 0.7228647470474243,
+      "learning_rate": 7.316469617816882e-07,
+      "loss": 1.117,
+      "mean_token_accuracy": 0.6872781366109848,
+      "num_tokens": 1656064.0,
+      "step": 1090
+    },
+    {
+      "entropy": 1.1704509019851685,
+      "epoch": 0.88,
+      "grad_norm": 0.6274661421775818,
+      "learning_rate": 7.288974429474841e-07,
+      "loss": 1.0567,
+      "mean_token_accuracy": 0.6986218981444836,
+      "num_tokens": 1671905.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.1555724613368512,
+      "epoch": 0.888,
+      "grad_norm": 0.5717065930366516,
+      "learning_rate": 7.261479241132802e-07,
+      "loss": 1.0613,
+      "mean_token_accuracy": 0.6985699016600847,
+      "num_tokens": 1687831.0,
+      "step": 1110
+    },
+    {
+      "entropy": 1.1825853794813157,
+      "epoch": 0.896,
+      "grad_norm": 0.5806442499160767,
+      "learning_rate": 7.233984052790761e-07,
+      "loss": 1.1196,
+      "mean_token_accuracy": 0.6930595509707927,
+      "num_tokens": 1703242.0,
+      "step": 1120
+    },
+    {
+      "entropy": 1.1857046082615852,
+      "epoch": 0.904,
+      "grad_norm": 0.7729993462562561,
+      "learning_rate": 7.206488864448721e-07,
+      "loss": 1.0937,
+      "mean_token_accuracy": 0.692601241916418,
+      "num_tokens": 1717420.0,
+      "step": 1130
+    },
+    {
+      "entropy": 1.15954297054559,
+      "epoch": 0.912,
+      "grad_norm": 0.5793495178222656,
+      "learning_rate": 7.178993676106681e-07,
+      "loss": 1.0322,
+      "mean_token_accuracy": 0.7003567360341549,
+      "num_tokens": 1734570.0,
+      "step": 1140
+    },
+    {
+      "entropy": 1.1955730587244033,
+      "epoch": 0.92,
+      "grad_norm": 0.6606018543243408,
+      "learning_rate": 7.151498487764641e-07,
+      "loss": 1.1115,
+      "mean_token_accuracy": 0.6888970002532006,
+      "num_tokens": 1749977.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.1169864691793918,
+      "epoch": 0.928,
+      "grad_norm": 0.511619508266449,
+      "learning_rate": 7.124003299422601e-07,
+      "loss": 1.0671,
+      "mean_token_accuracy": 0.7025054290890693,
+      "num_tokens": 1766717.0,
+      "step": 1160
+    },
+    {
+      "entropy": 1.145845735212788,
+      "epoch": 0.936,
+      "grad_norm": 0.7826500535011292,
+      "learning_rate": 7.09650811108056e-07,
+      "loss": 1.0499,
+      "mean_token_accuracy": 0.6996002644300461,
+      "num_tokens": 1781248.0,
+      "step": 1170
+    },
+    {
+      "entropy": 1.1137450352311133,
+      "epoch": 0.944,
+      "grad_norm": 0.6092593669891357,
+      "learning_rate": 7.069012922738521e-07,
+      "loss": 1.0631,
+      "mean_token_accuracy": 0.7020095460116863,
+      "num_tokens": 1797901.0,
+      "step": 1180
+    },
+    {
+      "entropy": 1.2010398358106613,
+      "epoch": 0.952,
+      "grad_norm": 0.9775738716125488,
+      "learning_rate": 7.04151773439648e-07,
+      "loss": 1.1548,
+      "mean_token_accuracy": 0.6882500603795052,
+      "num_tokens": 1811944.0,
+      "step": 1190
+    },
+    {
+      "entropy": 1.1422110214829444,
+      "epoch": 0.96,
+      "grad_norm": 0.715650737285614,
+      "learning_rate": 7.01402254605444e-07,
+      "loss": 1.1148,
+      "mean_token_accuracy": 0.6927286379039288,
+      "num_tokens": 1828178.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.14734987616539,
+      "epoch": 0.968,
+      "grad_norm": 0.5900126099586487,
+      "learning_rate": 6.9865273577124e-07,
+      "loss": 1.0809,
+      "mean_token_accuracy": 0.6929037302732468,
+      "num_tokens": 1845334.0,
+      "step": 1210
+    },
+    {
+      "entropy": 1.1666462287306785,
+      "epoch": 0.976,
+      "grad_norm": 0.7052849531173706,
+      "learning_rate": 6.95903216937036e-07,
+      "loss": 1.086,
+      "mean_token_accuracy": 0.6887565299868583,
+      "num_tokens": 1859542.0,
+      "step": 1220
+    },
+    {
+      "entropy": 1.127211359143257,
+      "epoch": 0.984,
+      "grad_norm": 0.9729690551757812,
+      "learning_rate": 6.93153698102832e-07,
+      "loss": 1.0588,
+      "mean_token_accuracy": 0.69860158264637,
+      "num_tokens": 1876209.0,
+      "step": 1230
+    },
+    {
+      "entropy": 1.1535819880664349,
+      "epoch": 0.992,
+      "grad_norm": 0.6623355746269226,
+      "learning_rate": 6.904041792686279e-07,
+      "loss": 1.0543,
+      "mean_token_accuracy": 0.695023113116622,
+      "num_tokens": 1893007.0,
+      "step": 1240
+    },
+    {
+      "entropy": 1.1296846129000186,
+      "epoch": 1.0,
+      "grad_norm": 0.6806105375289917,
+      "learning_rate": 6.87654660434424e-07,
+      "loss": 1.089,
+      "mean_token_accuracy": 0.6975006379187108,
+      "num_tokens": 1908628.0,
+      "step": 1250
+    },
+    {
+      "entropy": 1.1887501999735832,
+      "epoch": 1.008,
+      "grad_norm": 0.6205907464027405,
+      "learning_rate": 6.849051416002199e-07,
+      "loss": 1.1017,
+      "mean_token_accuracy": 0.695924699306488,
+      "num_tokens": 1923760.0,
+      "step": 1260
+    },
+    {
+      "entropy": 1.1948752962052822,
+      "epoch": 1.016,
+      "grad_norm": 0.6140705943107605,
+      "learning_rate": 6.821556227660159e-07,
+      "loss": 1.1083,
+      "mean_token_accuracy": 0.6905615709722042,
+      "num_tokens": 1938046.0,
+      "step": 1270
+    },
+    {
+      "entropy": 1.19633205011487,
+      "epoch": 1.024,
+      "grad_norm": 0.6875778436660767,
+      "learning_rate": 6.794061039318119e-07,
+      "loss": 1.0815,
+      "mean_token_accuracy": 0.6873594917356968,
+      "num_tokens": 1952387.0,
+      "step": 1280
+    },
+    {
+      "entropy": 1.1888278536498547,
+      "epoch": 1.032,
+      "grad_norm": 0.6977635622024536,
+      "learning_rate": 6.766565850976079e-07,
+      "loss": 1.1028,
+      "mean_token_accuracy": 0.6910716913640499,
+      "num_tokens": 1968460.0,
+      "step": 1290
+    },
+    {
+      "entropy": 1.136767565459013,
+      "epoch": 1.04,
+      "grad_norm": 0.9059696197509766,
+      "learning_rate": 6.739070662634039e-07,
+      "loss": 1.0709,
+      "mean_token_accuracy": 0.6994237117469311,
+      "num_tokens": 1983262.0,
+      "step": 1300
+    },
+    {
+      "entropy": 1.1318768605589866,
+      "epoch": 1.048,
+      "grad_norm": 0.6679156422615051,
+      "learning_rate": 6.711575474291998e-07,
+      "loss": 1.0656,
+      "mean_token_accuracy": 0.7010302551090717,
+      "num_tokens": 1998424.0,
+      "step": 1310
+    },
+    {
+      "entropy": 1.1715723387897015,
+      "epoch": 1.056,
+      "grad_norm": 0.7726727724075317,
+      "learning_rate": 6.684080285949959e-07,
+      "loss": 1.1207,
+      "mean_token_accuracy": 0.6910806596279144,
+      "num_tokens": 2012814.0,
+      "step": 1320
+    },
+    {
+      "entropy": 1.1851845063269137,
+      "epoch": 1.064,
+      "grad_norm": 0.7148599028587341,
+      "learning_rate": 6.656585097607918e-07,
+      "loss": 1.1488,
+      "mean_token_accuracy": 0.6808865670114755,
+      "num_tokens": 2027608.0,
+      "step": 1330
+    },
+    {
+      "entropy": 1.1245805986225605,
+      "epoch": 1.072,
+      "grad_norm": 0.4553343951702118,
+      "learning_rate": 6.629089909265878e-07,
+      "loss": 1.0839,
+      "mean_token_accuracy": 0.697356453537941,
+      "num_tokens": 2045654.0,
+      "step": 1340
+    },
+    {
+      "entropy": 1.1225988395512103,
+      "epoch": 1.08,
+      "grad_norm": 0.7872028350830078,
+      "learning_rate": 6.601594720923838e-07,
+      "loss": 1.0784,
+      "mean_token_accuracy": 0.6945089906454086,
+      "num_tokens": 2060994.0,
+      "step": 1350
+    },
+    {
+      "entropy": 1.1490298748016357,
+      "epoch": 1.088,
+      "grad_norm": 0.8394035696983337,
+      "learning_rate": 6.574099532581798e-07,
+      "loss": 1.1063,
+      "mean_token_accuracy": 0.6996151521801949,
+      "num_tokens": 2075005.0,
+      "step": 1360
+    },
+    {
+      "entropy": 1.1323794476687907,
+      "epoch": 1.096,
+      "grad_norm": 0.605958878993988,
+      "learning_rate": 6.546604344239758e-07,
+      "loss": 1.0654,
+      "mean_token_accuracy": 0.7028530709445476,
+      "num_tokens": 2089435.0,
+      "step": 1370
+    },
+    {
+      "entropy": 1.112098726630211,
+      "epoch": 1.104,
+      "grad_norm": 0.584015965461731,
+      "learning_rate": 6.519109155897717e-07,
+      "loss": 1.0569,
+      "mean_token_accuracy": 0.6996716480702162,
+      "num_tokens": 2105434.0,
+      "step": 1380
+    },
+    {
+      "entropy": 1.1244696512818337,
+      "epoch": 1.112,
+      "grad_norm": 0.6987764239311218,
+      "learning_rate": 6.491613967555677e-07,
+      "loss": 1.0595,
+      "mean_token_accuracy": 0.7015144415199757,
+      "num_tokens": 2120817.0,
+      "step": 1390
+    },
+    {
+      "entropy": 1.1288893558084965,
+      "epoch": 1.12,
+      "grad_norm": 0.4876547157764435,
+      "learning_rate": 6.464118779213637e-07,
+      "loss": 1.0409,
+      "mean_token_accuracy": 0.6968514807522297,
+      "num_tokens": 2136838.0,
+      "step": 1400
+    },
+    {
+      "entropy": 1.1660522818565369,
+      "epoch": 1.1280000000000001,
+      "grad_norm": 0.6854759454727173,
+      "learning_rate": 6.436623590871597e-07,
+      "loss": 1.1312,
+      "mean_token_accuracy": 0.6878326304256916,
+      "num_tokens": 2151350.0,
+      "step": 1410
+    },
+    {
+      "entropy": 1.1638130411505698,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 0.9255546927452087,
+      "learning_rate": 6.409128402529557e-07,
+      "loss": 1.1307,
+      "mean_token_accuracy": 0.6938613057136536,
+      "num_tokens": 2166279.0,
+      "step": 1420
+    },
+    {
+      "entropy": 1.1093559078872204,
+      "epoch": 1.144,
+      "grad_norm": 0.6495608687400818,
+      "learning_rate": 6.381633214187517e-07,
+      "loss": 1.0351,
+      "mean_token_accuracy": 0.7085234835743904,
+      "num_tokens": 2181882.0,
+      "step": 1430
+    },
+    {
+      "entropy": 1.1332037717103958,
+      "epoch": 1.152,
+      "grad_norm": 0.6880368590354919,
+      "learning_rate": 6.354138025845477e-07,
+      "loss": 1.0714,
+      "mean_token_accuracy": 0.6985657840967179,
+      "num_tokens": 2198399.0,
+      "step": 1440
+    },
+    {
+      "entropy": 1.1295147154480218,
+      "epoch": 1.16,
+      "grad_norm": 0.8683180809020996,
+      "learning_rate": 6.326642837503436e-07,
+      "loss": 1.0736,
+      "mean_token_accuracy": 0.6970417097210884,
+      "num_tokens": 2212988.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.1400238119065762,
+      "epoch": 1.168,
+      "grad_norm": 0.691609799861908,
+      "learning_rate": 6.299147649161396e-07,
+      "loss": 1.097,
+      "mean_token_accuracy": 0.6984522871673107,
+      "num_tokens": 2227368.0,
+      "step": 1460
+    },
+    {
+      "entropy": 1.1733529463410377,
+      "epoch": 1.176,
+      "grad_norm": 0.9658370614051819,
+      "learning_rate": 6.271652460819357e-07,
+      "loss": 1.1164,
+      "mean_token_accuracy": 0.6877712234854698,
+      "num_tokens": 2242927.0,
+      "step": 1470
+    },
+    {
+      "entropy": 1.137531255185604,
+      "epoch": 1.184,
+      "grad_norm": 0.8065728545188904,
+      "learning_rate": 6.244157272477316e-07,
+      "loss": 1.0707,
+      "mean_token_accuracy": 0.6989330679178238,
+      "num_tokens": 2256714.0,
+      "step": 1480
+    },
+    {
+      "entropy": 1.1494978316128255,
+      "epoch": 1.192,
+      "grad_norm": 0.629590630531311,
+      "learning_rate": 6.216662084135276e-07,
+      "loss": 1.098,
+      "mean_token_accuracy": 0.6965864278376103,
+      "num_tokens": 2270629.0,
+      "step": 1490
+    },
+    {
+      "entropy": 1.159467339515686,
+      "epoch": 1.2,
+      "grad_norm": 0.686192512512207,
+      "learning_rate": 6.189166895793236e-07,
+      "loss": 1.0893,
+      "mean_token_accuracy": 0.6922002293169498,
+      "num_tokens": 2288460.0,
+      "step": 1500
+    },
+    {
+      "entropy": 1.0993162184953689,
+      "epoch": 1.208,
+      "grad_norm": 0.9897556304931641,
+      "learning_rate": 6.161671707451196e-07,
+      "loss": 1.0455,
+      "mean_token_accuracy": 0.705514881759882,
+      "num_tokens": 2302251.0,
+      "step": 1510
+    },
+    {
+      "entropy": 1.12115698158741,
+      "epoch": 1.216,
+      "grad_norm": 0.7470293641090393,
+      "learning_rate": 6.134176519109155e-07,
+      "loss": 1.0789,
+      "mean_token_accuracy": 0.6973060339689254,
+      "num_tokens": 2318753.0,
+      "step": 1520
+    },
+    {
+      "entropy": 1.1007006213068962,
+      "epoch": 1.224,
+      "grad_norm": 0.8028280735015869,
+      "learning_rate": 6.106681330767115e-07,
+      "loss": 1.0386,
+      "mean_token_accuracy": 0.7066539935767651,
+      "num_tokens": 2334923.0,
+      "step": 1530
+    },
+    {
+      "entropy": 1.1649780459702015,
+      "epoch": 1.232,
+      "grad_norm": 0.6267876625061035,
+      "learning_rate": 6.079186142425076e-07,
+      "loss": 1.0524,
+      "mean_token_accuracy": 0.6983841970562935,
+      "num_tokens": 2349793.0,
+      "step": 1540
+    },
+    {
+      "entropy": 1.0742624267935752,
+      "epoch": 1.24,
+      "grad_norm": 0.554948091506958,
+      "learning_rate": 6.051690954083035e-07,
+      "loss": 1.059,
+      "mean_token_accuracy": 0.7093812972307205,
+      "num_tokens": 2365509.0,
+      "step": 1550
+    },
+    {
+      "entropy": 1.0984411075711251,
+      "epoch": 1.248,
+      "grad_norm": 0.9897919297218323,
+      "learning_rate": 6.024195765740995e-07,
+      "loss": 1.0389,
+      "mean_token_accuracy": 0.7038175620138645,
+      "num_tokens": 2381097.0,
+      "step": 1560
+    },
+    {
+      "entropy": 1.1904704883694648,
+      "epoch": 1.256,
+      "grad_norm": 0.7495393753051758,
+      "learning_rate": 5.996700577398955e-07,
+      "loss": 1.1495,
+      "mean_token_accuracy": 0.6790803030133248,
+      "num_tokens": 2394963.0,
+      "step": 1570
+    },
+    {
+      "entropy": 1.124031263962388,
+      "epoch": 1.264,
+      "grad_norm": 0.7549392580986023,
+      "learning_rate": 5.969205389056915e-07,
+      "loss": 1.0414,
+      "mean_token_accuracy": 0.7057827066630125,
+      "num_tokens": 2408635.0,
+      "step": 1580
+    },
+    {
+      "entropy": 1.1567617639899255,
+      "epoch": 1.272,
+      "grad_norm": 0.7392032146453857,
+      "learning_rate": 5.941710200714874e-07,
+      "loss": 1.0694,
+      "mean_token_accuracy": 0.6952364321798086,
+      "num_tokens": 2423939.0,
+      "step": 1590
+    },
+    {
+      "entropy": 1.1447319120168686,
+      "epoch": 1.28,
+      "grad_norm": 0.6876333355903625,
+      "learning_rate": 5.914215012372834e-07,
+      "loss": 1.0857,
+      "mean_token_accuracy": 0.701924566924572,
+      "num_tokens": 2439207.0,
+      "step": 1600
+    },
+    {
+      "entropy": 1.1182593934237957,
+      "epoch": 1.288,
+      "grad_norm": 0.6917943358421326,
+      "learning_rate": 5.886719824030795e-07,
+      "loss": 1.0596,
+      "mean_token_accuracy": 0.7057459600269794,
+      "num_tokens": 2453639.0,
+      "step": 1610
+    },
+    {
+      "entropy": 1.1037614174187182,
+      "epoch": 1.296,
+      "grad_norm": 0.624317467212677,
+      "learning_rate": 5.859224635688754e-07,
+      "loss": 1.0565,
+      "mean_token_accuracy": 0.7065452396869659,
+      "num_tokens": 2468998.0,
+      "step": 1620
+    },
+    {
+      "entropy": 1.154854678362608,
+      "epoch": 1.304,
+      "grad_norm": 0.7155218124389648,
+      "learning_rate": 5.831729447346714e-07,
+      "loss": 1.0843,
+      "mean_token_accuracy": 0.6975614242255688,
+      "num_tokens": 2483501.0,
+      "step": 1630
+    },
+    {
+      "entropy": 1.1638387076556682,
+      "epoch": 1.312,
+      "grad_norm": 0.6090760231018066,
+      "learning_rate": 5.804234259004674e-07,
+      "loss": 1.0636,
+      "mean_token_accuracy": 0.6995426420122385,
+      "num_tokens": 2498494.0,
+      "step": 1640
+    },
+    {
+      "entropy": 1.1397521741688252,
+      "epoch": 1.32,
+      "grad_norm": 0.6465908288955688,
+      "learning_rate": 5.776739070662634e-07,
+      "loss": 1.0672,
+      "mean_token_accuracy": 0.6970544293522835,
+      "num_tokens": 2513937.0,
+      "step": 1650
+    },
+    {
+      "entropy": 1.1726163610816003,
+      "epoch": 1.328,
+      "grad_norm": 0.6399316787719727,
+      "learning_rate": 5.749243882320593e-07,
+      "loss": 1.097,
+      "mean_token_accuracy": 0.6957763388752938,
+      "num_tokens": 2527596.0,
+      "step": 1660
+    },
+    {
+      "entropy": 1.1261462047696114,
+      "epoch": 1.336,
+      "grad_norm": 0.6095896363258362,
+      "learning_rate": 5.721748693978553e-07,
+      "loss": 1.0764,
+      "mean_token_accuracy": 0.7021488003432751,
+      "num_tokens": 2542075.0,
+      "step": 1670
+    },
+    {
+      "entropy": 1.120316968858242,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 0.9220680594444275,
+      "learning_rate": 5.694253505636514e-07,
+      "loss": 1.0582,
+      "mean_token_accuracy": 0.6989481635391712,
+      "num_tokens": 2556818.0,
+      "step": 1680
+    },
+    {
+      "entropy": 1.180855856835842,
+      "epoch": 1.3519999999999999,
+      "grad_norm": 0.8124385476112366,
+      "learning_rate": 5.666758317294473e-07,
+      "loss": 1.0902,
+      "mean_token_accuracy": 0.6903267130255699,
+      "num_tokens": 2570623.0,
+      "step": 1690
+    },
+    {
+      "entropy": 1.1440489910542966,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 0.7189605236053467,
+      "learning_rate": 5.639263128952433e-07,
+      "loss": 1.052,
+      "mean_token_accuracy": 0.6996049843728542,
+      "num_tokens": 2586779.0,
+      "step": 1700
+    },
+    {
+      "entropy": 1.1032312862575053,
+      "epoch": 1.3679999999999999,
+      "grad_norm": 0.7067267894744873,
+      "learning_rate": 5.611767940610392e-07,
+      "loss": 1.0499,
+      "mean_token_accuracy": 0.7084492161870003,
+      "num_tokens": 2601690.0,
+      "step": 1710
+    },
+    {
+      "entropy": 1.109426486492157,
+      "epoch": 1.376,
+      "grad_norm": 0.824908435344696,
+      "learning_rate": 5.584272752268353e-07,
+      "loss": 1.0541,
+      "mean_token_accuracy": 0.7122204676270485,
+      "num_tokens": 2615836.0,
+      "step": 1720
+    },
+    {
+      "entropy": 1.1011802315711976,
+      "epoch": 1.384,
+      "grad_norm": 0.7468234896659851,
+      "learning_rate": 5.556777563926312e-07,
+      "loss": 1.0223,
+      "mean_token_accuracy": 0.7043912798166275,
+      "num_tokens": 2630440.0,
+      "step": 1730
+    },
+    {
+      "entropy": 1.1109345614910127,
+      "epoch": 1.392,
+      "grad_norm": 0.721721887588501,
+      "learning_rate": 5.529282375584272e-07,
+      "loss": 1.0669,
+      "mean_token_accuracy": 0.6989217452704907,
+      "num_tokens": 2645934.0,
+      "step": 1740
+    },
+    {
+      "entropy": 1.0837133921682836,
+      "epoch": 1.4,
+      "grad_norm": 0.6159181594848633,
+      "learning_rate": 5.501787187242233e-07,
+      "loss": 1.0516,
+      "mean_token_accuracy": 0.7033235274255276,
+      "num_tokens": 2661853.0,
+      "step": 1750
+    },
+    {
+      "entropy": 1.1363109357655048,
+      "epoch": 1.408,
+      "grad_norm": 0.5618049502372742,
+      "learning_rate": 5.474291998900192e-07,
+      "loss": 1.0491,
+      "mean_token_accuracy": 0.7057520292699337,
+      "num_tokens": 2676081.0,
+      "step": 1760
+    },
+    {
+      "entropy": 1.0844260066747666,
+      "epoch": 1.416,
+      "grad_norm": 0.5836204290390015,
+      "learning_rate": 5.446796810558152e-07,
+      "loss": 1.0251,
+      "mean_token_accuracy": 0.7037106763571501,
+      "num_tokens": 2693603.0,
+      "step": 1770
+    },
+    {
+      "entropy": 1.1237892121076585,
+      "epoch": 1.424,
+      "grad_norm": 0.8310570120811462,
+      "learning_rate": 5.419301622216111e-07,
+      "loss": 1.0645,
+      "mean_token_accuracy": 0.7011990338563919,
+      "num_tokens": 2709643.0,
+      "step": 1780
+    },
+    {
+      "entropy": 1.0602977596223355,
+      "epoch": 1.432,
+      "grad_norm": 0.5430757999420166,
+      "learning_rate": 5.391806433874072e-07,
+      "loss": 1.0362,
+      "mean_token_accuracy": 0.7056897468864918,
+      "num_tokens": 2727088.0,
+      "step": 1790
+    },
+    {
+      "entropy": 1.1189053302630783,
+      "epoch": 1.44,
+      "grad_norm": 0.8106410503387451,
+      "learning_rate": 5.364311245532031e-07,
+      "loss": 1.0914,
+      "mean_token_accuracy": 0.695854177325964,
+      "num_tokens": 2742220.0,
+      "step": 1800
+    },
+    {
+      "entropy": 1.1349434733390809,
+      "epoch": 1.448,
+      "grad_norm": 0.7153880000114441,
+      "learning_rate": 5.336816057189991e-07,
+      "loss": 1.0422,
+      "mean_token_accuracy": 0.6948840513825416,
+      "num_tokens": 2758176.0,
+      "step": 1810
+    },
+    {
+      "entropy": 1.1447625994682311,
+      "epoch": 1.456,
+      "grad_norm": 0.5748754739761353,
+      "learning_rate": 5.309320868847952e-07,
+      "loss": 1.0695,
+      "mean_token_accuracy": 0.6957199551165104,
+      "num_tokens": 2774066.0,
+      "step": 1820
+    },
+    {
+      "entropy": 1.1320161737501622,
+      "epoch": 1.464,
+      "grad_norm": 0.7259387969970703,
+      "learning_rate": 5.281825680505911e-07,
+      "loss": 1.0882,
+      "mean_token_accuracy": 0.6841345451772213,
+      "num_tokens": 2790132.0,
+      "step": 1830
+    },
+    {
+      "entropy": 1.1178909540176392,
+      "epoch": 1.472,
+      "grad_norm": 0.6329604387283325,
+      "learning_rate": 5.254330492163871e-07,
+      "loss": 1.0691,
+      "mean_token_accuracy": 0.6973642166703939,
+      "num_tokens": 2805381.0,
+      "step": 1840
+    },
+    {
+      "entropy": 1.2120650928467511,
+      "epoch": 1.48,
+      "grad_norm": 0.6683312654495239,
+      "learning_rate": 5.22683530382183e-07,
+      "loss": 1.1063,
+      "mean_token_accuracy": 0.6968633852899074,
+      "num_tokens": 2818429.0,
+      "step": 1850
+    },
+    {
+      "entropy": 1.1095169879496098,
+      "epoch": 1.488,
+      "grad_norm": 0.6347110271453857,
+      "learning_rate": 5.199340115479791e-07,
+      "loss": 1.0658,
+      "mean_token_accuracy": 0.704700855910778,
+      "num_tokens": 2834690.0,
+      "step": 1860
+    },
+    {
+      "entropy": 1.116725465655327,
+      "epoch": 1.496,
+      "grad_norm": 0.6903342604637146,
+      "learning_rate": 5.17184492713775e-07,
+      "loss": 1.0555,
+      "mean_token_accuracy": 0.7035901382565498,
+      "num_tokens": 2851736.0,
+      "step": 1870
+    },
+    {
+      "entropy": 1.1211778946220874,
+      "epoch": 1.504,
+      "grad_norm": 0.6732014417648315,
+      "learning_rate": 5.14434973879571e-07,
+      "loss": 1.0562,
+      "mean_token_accuracy": 0.696921756863594,
+      "num_tokens": 2866350.0,
+      "step": 1880
+    },
+    {
+      "entropy": 1.1037423841655254,
+      "epoch": 1.512,
+      "grad_norm": 0.6821587085723877,
+      "learning_rate": 5.116854550453671e-07,
+      "loss": 1.0714,
+      "mean_token_accuracy": 0.6999694172292947,
+      "num_tokens": 2882047.0,
+      "step": 1890
+    },
+    {
+      "entropy": 1.1391877956688403,
+      "epoch": 1.52,
+      "grad_norm": 0.6841466426849365,
+      "learning_rate": 5.08935936211163e-07,
+      "loss": 1.0704,
+      "mean_token_accuracy": 0.7005617998540401,
+      "num_tokens": 2898573.0,
+      "step": 1900
+    },
+    {
+      "entropy": 1.1329809248447418,
+      "epoch": 1.528,
+      "grad_norm": 0.7017571330070496,
+      "learning_rate": 5.06186417376959e-07,
+      "loss": 1.0147,
+      "mean_token_accuracy": 0.702555438131094,
+      "num_tokens": 2914374.0,
+      "step": 1910
+    },
+    {
+      "entropy": 1.1154800534248352,
+      "epoch": 1.536,
+      "grad_norm": 0.6277393102645874,
+      "learning_rate": 5.034368985427549e-07,
+      "loss": 1.0869,
+      "mean_token_accuracy": 0.7051818937063217,
+      "num_tokens": 2929124.0,
+      "step": 1920
+    },
+    {
+      "entropy": 1.0733695082366466,
+      "epoch": 1.544,
+      "grad_norm": 0.571405827999115,
+      "learning_rate": 5.00687379708551e-07,
+      "loss": 1.0056,
+      "mean_token_accuracy": 0.7166364155709743,
+      "num_tokens": 2944094.0,
+      "step": 1930
+    },
+    {
+      "entropy": 1.0417616236954927,
+      "epoch": 1.552,
+      "grad_norm": 0.5355597138404846,
+      "learning_rate": 4.979378608743469e-07,
+      "loss": 0.9785,
+      "mean_token_accuracy": 0.7171097300946713,
+      "num_tokens": 2960076.0,
+      "step": 1940
+    },
+    {
+      "entropy": 1.1525820292532445,
+      "epoch": 1.56,
+      "grad_norm": 0.7186400890350342,
+      "learning_rate": 4.951883420401429e-07,
+      "loss": 1.0642,
+      "mean_token_accuracy": 0.6978728592395782,
+      "num_tokens": 2975251.0,
+      "step": 1950
+    },
+    {
+      "entropy": 1.0816805481910705,
+      "epoch": 1.568,
+      "grad_norm": 0.4818354547023773,
+      "learning_rate": 4.92438823205939e-07,
+      "loss": 1.0038,
+      "mean_token_accuracy": 0.7125628098845482,
+      "num_tokens": 2991605.0,
+      "step": 1960
+    },
+    {
+      "entropy": 1.1187580041587353,
+      "epoch": 1.576,
+      "grad_norm": 0.810943603515625,
+      "learning_rate": 4.896893043717349e-07,
+      "loss": 1.047,
+      "mean_token_accuracy": 0.7004340924322605,
+      "num_tokens": 3008005.0,
+      "step": 1970
+    },
+    {
+      "entropy": 1.138374811410904,
+      "epoch": 1.584,
+      "grad_norm": 0.6631013751029968,
+      "learning_rate": 4.869397855375309e-07,
+      "loss": 1.0632,
+      "mean_token_accuracy": 0.698732690513134,
+      "num_tokens": 3023212.0,
+      "step": 1980
+    },
+    {
+      "entropy": 1.114051354676485,
+      "epoch": 1.592,
+      "grad_norm": 0.7821894288063049,
+      "learning_rate": 4.841902667033269e-07,
+      "loss": 1.0575,
+      "mean_token_accuracy": 0.702291352301836,
+      "num_tokens": 3038609.0,
+      "step": 1990
+    },
+    {
+      "entropy": 1.1360465161502362,
+      "epoch": 1.6,
+      "grad_norm": 0.9878914952278137,
+      "learning_rate": 4.814407478691229e-07,
+      "loss": 1.0268,
+      "mean_token_accuracy": 0.7054705023765564,
+      "num_tokens": 3053467.0,
+      "step": 2000
+    },
+    {
+      "entropy": 1.11304831802845,
+      "epoch": 1.608,
+      "grad_norm": 0.8406100273132324,
+      "learning_rate": 4.786912290349188e-07,
+      "loss": 1.0683,
+      "mean_token_accuracy": 0.7072804719209671,
+      "num_tokens": 3068178.0,
+      "step": 2010
+    },
+    {
+      "entropy": 1.082274530827999,
+      "epoch": 1.616,
+      "grad_norm": 0.5867545008659363,
+      "learning_rate": 4.7594171020071485e-07,
+      "loss": 1.0246,
+      "mean_token_accuracy": 0.7147180207073689,
+      "num_tokens": 3083931.0,
+      "step": 2020
+    },
+    {
+      "entropy": 1.1067815896123647,
+      "epoch": 1.624,
+      "grad_norm": 0.5816494226455688,
+      "learning_rate": 4.7319219136651086e-07,
+      "loss": 1.071,
+      "mean_token_accuracy": 0.7042274229228497,
+      "num_tokens": 3099164.0,
+      "step": 2030
+    },
+    {
+      "entropy": 1.1147871725261211,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 0.7727411389350891,
+      "learning_rate": 4.7044267253230687e-07,
+      "loss": 1.1101,
+      "mean_token_accuracy": 0.6992426700890064,
+      "num_tokens": 3116160.0,
+      "step": 2040
+    },
+    {
+      "entropy": 1.0435878798365592,
+      "epoch": 1.6400000000000001,
+      "grad_norm": 1.1286250352859497,
+      "learning_rate": 4.6769315369810283e-07,
+      "loss": 1.0239,
+      "mean_token_accuracy": 0.717649357765913,
+      "num_tokens": 3132039.0,
+      "step": 2050
+    },
+    {
+      "entropy": 1.1062811478972434,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 0.5837299823760986,
+      "learning_rate": 4.6494363486389884e-07,
+      "loss": 1.0199,
+      "mean_token_accuracy": 0.7072340235114097,
+      "num_tokens": 3147433.0,
+      "step": 2060
+    },
+    {
+      "entropy": 1.0895413413643837,
+      "epoch": 1.6560000000000001,
+      "grad_norm": 0.8702951073646545,
+      "learning_rate": 4.621941160296948e-07,
+      "loss": 1.0015,
+      "mean_token_accuracy": 0.7078986577689648,
+      "num_tokens": 3163993.0,
+      "step": 2070
+    },
+    {
+      "entropy": 1.0758908623829484,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 0.7220586538314819,
+      "learning_rate": 4.594445971954908e-07,
+      "loss": 0.9789,
+      "mean_token_accuracy": 0.7201415523886681,
+      "num_tokens": 3179094.0,
+      "step": 2080
+    },
+    {
+      "entropy": 1.1080400116741658,
+      "epoch": 1.6720000000000002,
+      "grad_norm": 0.8007322549819946,
+      "learning_rate": 4.5669507836128676e-07,
+      "loss": 1.0313,
+      "mean_token_accuracy": 0.7011715166270733,
+      "num_tokens": 3194939.0,
+      "step": 2090
+    },
+    {
+      "entropy": 1.0932671882212162,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 0.7251306772232056,
+      "learning_rate": 4.5394555952708276e-07,
+      "loss": 1.0389,
+      "mean_token_accuracy": 0.7085845321416855,
+      "num_tokens": 3211267.0,
+      "step": 2100
+    },
+    {
+      "entropy": 1.1485629804432391,
+      "epoch": 1.688,
+      "grad_norm": 0.764833927154541,
+      "learning_rate": 4.511960406928787e-07,
+      "loss": 1.0857,
+      "mean_token_accuracy": 0.6940891742706299,
+      "num_tokens": 3225940.0,
+      "step": 2110
+    },
+    {
+      "entropy": 1.1061964988708497,
+      "epoch": 1.696,
+      "grad_norm": 0.8021067380905151,
+      "learning_rate": 4.4844652185867473e-07,
+      "loss": 1.0482,
+      "mean_token_accuracy": 0.7028830051422119,
+      "num_tokens": 3240921.0,
+      "step": 2120
+    },
+    {
+      "entropy": 1.0867221042513848,
+      "epoch": 1.704,
+      "grad_norm": 0.6969442367553711,
+      "learning_rate": 4.4569700302447074e-07,
+      "loss": 1.045,
+      "mean_token_accuracy": 0.7050205707550049,
+      "num_tokens": 3256275.0,
+      "step": 2130
+    },
+    {
+      "entropy": 1.1240694530308246,
+      "epoch": 1.712,
+      "grad_norm": 0.7253401875495911,
+      "learning_rate": 4.429474841902667e-07,
+      "loss": 1.0878,
+      "mean_token_accuracy": 0.7003125563263893,
+      "num_tokens": 3270866.0,
+      "step": 2140
+    },
+    {
+      "entropy": 1.088991916924715,
+      "epoch": 1.72,
+      "grad_norm": 0.8315143585205078,
+      "learning_rate": 4.401979653560627e-07,
+      "loss": 1.0295,
+      "mean_token_accuracy": 0.7133419007062912,
+      "num_tokens": 3286070.0,
+      "step": 2150
+    },
+    {
+      "entropy": 1.127638630196452,
+      "epoch": 1.728,
+      "grad_norm": 0.5839968323707581,
+      "learning_rate": 4.3744844652185866e-07,
+      "loss": 1.0575,
+      "mean_token_accuracy": 0.6951487042009831,
+      "num_tokens": 3300179.0,
+      "step": 2160
+    },
+    {
+      "entropy": 1.0937600523233413,
+      "epoch": 1.736,
+      "grad_norm": 0.9718707203865051,
+      "learning_rate": 4.3469892768765467e-07,
+      "loss": 1.0645,
+      "mean_token_accuracy": 0.7096135891973973,
+      "num_tokens": 3314060.0,
+      "step": 2170
+    },
+    {
+      "entropy": 1.1053045496344567,
+      "epoch": 1.744,
+      "grad_norm": 0.615055501461029,
+      "learning_rate": 4.319494088534506e-07,
+      "loss": 1.0444,
+      "mean_token_accuracy": 0.7076170362532139,
+      "num_tokens": 3329115.0,
+      "step": 2180
+    },
+    {
+      "entropy": 1.1166148386895656,
+      "epoch": 1.752,
+      "grad_norm": 0.9525709748268127,
+      "learning_rate": 4.2919989001924663e-07,
+      "loss": 1.0631,
+      "mean_token_accuracy": 0.7037151999771595,
+      "num_tokens": 3345499.0,
+      "step": 2190
+    },
+    {
+      "entropy": 1.1394428189843893,
+      "epoch": 1.76,
+      "grad_norm": 0.6527174115180969,
+      "learning_rate": 4.2645037118504264e-07,
+      "loss": 1.0539,
+      "mean_token_accuracy": 0.6947876572608948,
+      "num_tokens": 3361802.0,
+      "step": 2200
+    },
+    {
+      "entropy": 1.0809528261423111,
+      "epoch": 1.768,
+      "grad_norm": 0.8444985151290894,
+      "learning_rate": 4.237008523508386e-07,
+      "loss": 0.9814,
+      "mean_token_accuracy": 0.7183415681123734,
+      "num_tokens": 3376707.0,
+      "step": 2210
+    },
+    {
+      "entropy": 1.135873431712389,
+      "epoch": 1.776,
+      "grad_norm": 0.683020830154419,
+      "learning_rate": 4.209513335166346e-07,
+      "loss": 1.0697,
+      "mean_token_accuracy": 0.6985464196652174,
+      "num_tokens": 3390253.0,
+      "step": 2220
+    },
+    {
+      "entropy": 1.1180547427386045,
+      "epoch": 1.784,
+      "grad_norm": 0.7533634305000305,
+      "learning_rate": 4.1820181468243056e-07,
+      "loss": 1.044,
+      "mean_token_accuracy": 0.7064264304935932,
+      "num_tokens": 3405779.0,
+      "step": 2230
+    },
+    {
+      "entropy": 1.0656680211424827,
+      "epoch": 1.792,
+      "grad_norm": 0.536811113357544,
+      "learning_rate": 4.1545229584822657e-07,
+      "loss": 1.0282,
+      "mean_token_accuracy": 0.721583554893732,
+      "num_tokens": 3422545.0,
+      "step": 2240
+    },
+    {
+      "entropy": 1.110101143270731,
+      "epoch": 1.8,
+      "grad_norm": 0.5704901814460754,
+      "learning_rate": 4.127027770140225e-07,
+      "loss": 1.0368,
+      "mean_token_accuracy": 0.7042762607336044,
+      "num_tokens": 3438250.0,
+      "step": 2250
+    },
+    {
+      "entropy": 1.116504292935133,
+      "epoch": 1.808,
+      "grad_norm": 0.6488993167877197,
+      "learning_rate": 4.0995325817981853e-07,
+      "loss": 1.067,
+      "mean_token_accuracy": 0.6993104033172131,
+      "num_tokens": 3453393.0,
+      "step": 2260
+    },
+    {
+      "entropy": 1.1124402858316897,
+      "epoch": 1.8159999999999998,
+      "grad_norm": 0.8528743386268616,
+      "learning_rate": 4.072037393456145e-07,
+      "loss": 1.0566,
+      "mean_token_accuracy": 0.700559002161026,
+      "num_tokens": 3467151.0,
+      "step": 2270
+    },
+    {
+      "entropy": 1.1406603172421454,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 0.7719080448150635,
+      "learning_rate": 4.044542205114105e-07,
+      "loss": 1.0755,
+      "mean_token_accuracy": 0.6937859803438187,
+      "num_tokens": 3482533.0,
+      "step": 2280
+    },
+    {
+      "entropy": 1.208981278538704,
+      "epoch": 1.8319999999999999,
+      "grad_norm": 0.6996201872825623,
+      "learning_rate": 4.017047016772065e-07,
+      "loss": 1.1165,
+      "mean_token_accuracy": 0.6901387263089418,
+      "num_tokens": 3496858.0,
+      "step": 2290
+    },
+    {
+      "entropy": 1.1587674543261528,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 0.6966138482093811,
+      "learning_rate": 3.9895518284300246e-07,
+      "loss": 1.0776,
+      "mean_token_accuracy": 0.6850869338959455,
+      "num_tokens": 3512387.0,
+      "step": 2300
+    },
+    {
+      "entropy": 1.0703238815069198,
+      "epoch": 1.8479999999999999,
+      "grad_norm": 0.7426791191101074,
+      "learning_rate": 3.9620566400879847e-07,
+      "loss": 0.9889,
+      "mean_token_accuracy": 0.7159933507442474,
+      "num_tokens": 3528509.0,
+      "step": 2310
+    },
+    {
+      "entropy": 1.0979916997253896,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 1.0224542617797852,
+      "learning_rate": 3.934561451745944e-07,
+      "loss": 1.0392,
+      "mean_token_accuracy": 0.704944996535778,
+      "num_tokens": 3543321.0,
+      "step": 2320
+    },
+    {
+      "entropy": 1.0528897911310195,
+      "epoch": 1.8639999999999999,
+      "grad_norm": 0.7673348188400269,
+      "learning_rate": 3.9070662634039043e-07,
+      "loss": 1.0468,
+      "mean_token_accuracy": 0.7065678708255291,
+      "num_tokens": 3558594.0,
+      "step": 2330
+    },
+    {
+      "entropy": 1.1163273386657238,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 0.7382457256317139,
+      "learning_rate": 3.879571075061864e-07,
+      "loss": 1.0612,
+      "mean_token_accuracy": 0.6982604905962944,
+      "num_tokens": 3574336.0,
+      "step": 2340
+    },
+    {
+      "entropy": 1.0881711460649968,
+      "epoch": 1.88,
+      "grad_norm": 0.6713865995407104,
+      "learning_rate": 3.852075886719824e-07,
+      "loss": 1.0252,
+      "mean_token_accuracy": 0.7134752497076988,
+      "num_tokens": 3588939.0,
+      "step": 2350
+    },
+    {
+      "entropy": 1.1154374856501819,
+      "epoch": 1.888,
+      "grad_norm": 0.973667562007904,
+      "learning_rate": 3.824580698377784e-07,
+      "loss": 1.0484,
+      "mean_token_accuracy": 0.6991922508925199,
+      "num_tokens": 3602925.0,
+      "step": 2360
+    },
+    {
+      "entropy": 1.0306460287421941,
+      "epoch": 1.896,
+      "grad_norm": 0.9138699769973755,
+      "learning_rate": 3.7970855100357436e-07,
+      "loss": 1.0054,
+      "mean_token_accuracy": 0.7150904729962348,
+      "num_tokens": 3619466.0,
+      "step": 2370
+    },
+    {
+      "entropy": 1.1330877557396888,
+      "epoch": 1.904,
+      "grad_norm": 0.6441805958747864,
+      "learning_rate": 3.7695903216937037e-07,
+      "loss": 1.0836,
+      "mean_token_accuracy": 0.7036154177039862,
+      "num_tokens": 3632130.0,
+      "step": 2380
+    },
+    {
+      "entropy": 1.0771817360073328,
+      "epoch": 1.912,
+      "grad_norm": 0.758843183517456,
+      "learning_rate": 3.742095133351663e-07,
+      "loss": 1.0124,
+      "mean_token_accuracy": 0.7171977959573269,
+      "num_tokens": 3647421.0,
+      "step": 2390
+    },
+    {
+      "entropy": 1.0749525152146817,
+      "epoch": 1.92,
+      "grad_norm": 0.6015713810920715,
+      "learning_rate": 3.7145999450096233e-07,
+      "loss": 1.0209,
+      "mean_token_accuracy": 0.7081304289400577,
+      "num_tokens": 3664479.0,
+      "step": 2400
+    },
+    {
+      "entropy": 1.097420910000801,
+      "epoch": 1.928,
+      "grad_norm": 0.8378422260284424,
+      "learning_rate": 3.687104756667583e-07,
+      "loss": 1.0119,
+      "mean_token_accuracy": 0.7116850987076759,
+      "num_tokens": 3679719.0,
+      "step": 2410
+    },
+    {
+      "entropy": 1.106580638512969,
+      "epoch": 1.936,
+      "grad_norm": 0.8262378573417664,
+      "learning_rate": 3.659609568325543e-07,
+      "loss": 1.0375,
+      "mean_token_accuracy": 0.708418320864439,
+      "num_tokens": 3694167.0,
+      "step": 2420
+    },
+    {
+      "entropy": 1.0778734140098094,
+      "epoch": 1.944,
+      "grad_norm": 0.854191780090332,
+      "learning_rate": 3.6321143799835025e-07,
+      "loss": 1.0133,
+      "mean_token_accuracy": 0.7053643822669983,
+      "num_tokens": 3709583.0,
+      "step": 2430
+    },
+    {
+      "entropy": 1.0541854061186313,
+      "epoch": 1.952,
+      "grad_norm": 0.7758670449256897,
+      "learning_rate": 3.6046191916414626e-07,
+      "loss": 1.0013,
+      "mean_token_accuracy": 0.715771260112524,
+      "num_tokens": 3724908.0,
+      "step": 2440
+    },
+    {
+      "entropy": 1.099145165970549,
+      "epoch": 1.96,
+      "grad_norm": 0.5164648294448853,
+      "learning_rate": 3.5771240032994227e-07,
+      "loss": 1.0649,
+      "mean_token_accuracy": 0.7001733347773552,
+      "num_tokens": 3740440.0,
+      "step": 2450
+    },
+    {
+      "entropy": 1.0599645994603635,
+      "epoch": 1.968,
+      "grad_norm": 0.5749976634979248,
+      "learning_rate": 3.5496288149573823e-07,
+      "loss": 1.0152,
+      "mean_token_accuracy": 0.711567934602499,
+      "num_tokens": 3757367.0,
+      "step": 2460
+    },
+    {
+      "entropy": 1.106277884170413,
+      "epoch": 1.976,
+      "grad_norm": 0.9316745400428772,
+      "learning_rate": 3.5221336266153424e-07,
+      "loss": 1.0279,
+      "mean_token_accuracy": 0.7048185043036937,
+      "num_tokens": 3772884.0,
+      "step": 2470
+    },
+    {
+      "entropy": 1.0862935207784177,
+      "epoch": 1.984,
+      "grad_norm": 1.0442012548446655,
+      "learning_rate": 3.494638438273302e-07,
+      "loss": 1.0653,
+      "mean_token_accuracy": 0.7030534707009792,
+      "num_tokens": 3788095.0,
+      "step": 2480
+    },
+    {
+      "entropy": 1.1105420462787152,
+      "epoch": 1.992,
+      "grad_norm": 0.6086395978927612,
+      "learning_rate": 3.467143249931262e-07,
+      "loss": 1.0192,
+      "mean_token_accuracy": 0.7073094092309475,
+      "num_tokens": 3802348.0,
+      "step": 2490
+    },
+    {
+      "entropy": 1.0711169321089984,
+      "epoch": 2.0,
+      "grad_norm": 0.7704114317893982,
+      "learning_rate": 3.4396480615892216e-07,
+      "loss": 1.0536,
+      "mean_token_accuracy": 0.7132571704685688,
+      "num_tokens": 3817256.0,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3750,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.549939873435484e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

full-lora/checkpoint-2500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:373064d186b379b705f3701bc2192b8d633e746e079c3d5e972608780158085a
+size 6417

full-lora/checkpoint-3750/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-2-7b-chat-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-2-7b-chat-hf
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

full-lora/checkpoint-3750/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

full-lora/checkpoint-3750/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23038c1e179be1b0cef2b41a2699c1eb17948c75072dcc812915cb90731e30eb
+size 639691872

full-lora/checkpoint-3750/chat_template.jinja

@@ -0,0 +1 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\n' + system_message + '\n<</SYS>>\n\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}

full-lora/checkpoint-3750/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07aeb20d6eb955c054fce3602440b51d8ed9573893ed5a991932e10839abe09b
+size 1279641507

full-lora/checkpoint-3750/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8889e9bf700264c2b7e4610f731b60c3bc83966c87a5b6c8f6dab9bdde52a8da
+size 14645

full-lora/checkpoint-3750/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f67c62618bce1bc4cbc50c8567f565ac7868ca695c9af4aceb09f3d8747526e1
+size 1465

full-lora/checkpoint-3750/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

full-lora/checkpoint-3750/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

full-lora/checkpoint-3750/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

full-lora/checkpoint-3750/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:373064d186b379b705f3701bc2192b8d633e746e079c3d5e972608780158085a
+size 6417

full-lora/run_config.json

@@ -0,0 +1,13 @@
+{
+  "model_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "num_train_epochs": 3.0,
+  "learning_rate": 1e-06,
+  "full_finetune": false,
+  "use_qlora": false,
+  "lora_r": 64,
+  "dataset": "HarethahMo/extended-refusal",
+  "max_steps": null,
+  "response_aspect": "full",
+  "train_examples": 10000,
+  "assistant_only_loss": false
+}

full-lora/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

full-lora/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

full-lora/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

full-lora/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:373064d186b379b705f3701bc2192b8d633e746e079c3d5e972608780158085a
+size 6417