Upload LoRA adapter for multitask text generation

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.ipynb_checkpoints/adapter_config-checkpoint.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen1.5-1.8B-Chat",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "q_proj",
+    "gate_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen1.5-1.8B-Chat
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen1.5-1.8B-Chat
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen1.5-1.8B-Chat",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "q_proj",
+    "gate_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2cb69f960dd557df9e3add198163eff66fdd84ea9126468788ee92f2227c0fd
+size 30026872

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful assistant.<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d213310c1c7d081361f0337b49ac89ff72d6c69c7e91aa2829e7d6eb9628e47
+size 16485827

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59fd62cc2ff6347d62b07e61ea3a7b13b9ecc8473cd85badcd11b0e6e8f26ce8
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:079e0e0ec8f2921a9bae691215c4eba44b0cda93f6f284269579a17bd94d7463
+size 1465

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9
+size 11418266

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,1834 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.6,
+  "eval_steps": 500,
+  "global_step": 9000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 2.334267578125,
+      "epoch": 0.0033333333333333335,
+      "grad_norm": 1.3027938604354858,
+      "learning_rate": 1.088888888888889e-05,
+      "loss": 3.2727,
+      "mean_token_accuracy": 0.37681460797786714,
+      "num_tokens": 152562.0,
+      "step": 50
+    },
+    {
+      "entropy": 2.6866015625,
+      "epoch": 0.006666666666666667,
+      "grad_norm": 1.0882155895233154,
+      "learning_rate": 2.2000000000000003e-05,
+      "loss": 2.9865,
+      "mean_token_accuracy": 0.4045091135799885,
+      "num_tokens": 305571.0,
+      "step": 100
+    },
+    {
+      "entropy": 2.694482421875,
+      "epoch": 0.01,
+      "grad_norm": 1.6767909526824951,
+      "learning_rate": 3.311111111111112e-05,
+      "loss": 2.8299,
+      "mean_token_accuracy": 0.4228819414973259,
+      "num_tokens": 458725.0,
+      "step": 150
+    },
+    {
+      "entropy": 2.666123046875,
+      "epoch": 0.013333333333333334,
+      "grad_norm": 1.948503017425537,
+      "learning_rate": 4.422222222222222e-05,
+      "loss": 2.7832,
+      "mean_token_accuracy": 0.4269903081655502,
+      "num_tokens": 611785.0,
+      "step": 200
+    },
+    {
+      "entropy": 2.7209375,
+      "epoch": 0.016666666666666666,
+      "grad_norm": 2.4782638549804688,
+      "learning_rate": 5.5333333333333334e-05,
+      "loss": 2.8287,
+      "mean_token_accuracy": 0.4207923886179924,
+      "num_tokens": 765026.0,
+      "step": 250
+    },
+    {
+      "entropy": 2.707451171875,
+      "epoch": 0.02,
+      "grad_norm": 2.3473691940307617,
+      "learning_rate": 6.644444444444444e-05,
+      "loss": 2.8256,
+      "mean_token_accuracy": 0.4248832304775715,
+      "num_tokens": 918026.0,
+      "step": 300
+    },
+    {
+      "entropy": 2.75130859375,
+      "epoch": 0.023333333333333334,
+      "grad_norm": 2.264916181564331,
+      "learning_rate": 7.755555555555556e-05,
+      "loss": 2.8603,
+      "mean_token_accuracy": 0.4202099771797657,
+      "num_tokens": 1071079.0,
+      "step": 350
+    },
+    {
+      "entropy": 2.7161962890625,
+      "epoch": 0.02666666666666667,
+      "grad_norm": 2.1681137084960938,
+      "learning_rate": 8.866666666666668e-05,
+      "loss": 2.8218,
+      "mean_token_accuracy": 0.4233370026946068,
+      "num_tokens": 1224719.0,
+      "step": 400
+    },
+    {
+      "entropy": 2.776904296875,
+      "epoch": 0.03,
+      "grad_norm": 2.9401867389678955,
+      "learning_rate": 9.977777777777779e-05,
+      "loss": 2.8903,
+      "mean_token_accuracy": 0.41743100568652153,
+      "num_tokens": 1377973.0,
+      "step": 450
+    },
+    {
+      "entropy": 2.729775390625,
+      "epoch": 0.03333333333333333,
+      "grad_norm": 2.047966480255127,
+      "learning_rate": 9.999720165100497e-05,
+      "loss": 2.8441,
+      "mean_token_accuracy": 0.4197780056297779,
+      "num_tokens": 1531040.0,
+      "step": 500
+    },
+    {
+      "entropy": 2.7143359375,
+      "epoch": 0.03666666666666667,
+      "grad_norm": 2.409952402114868,
+      "learning_rate": 9.998857733027178e-05,
+      "loss": 2.8122,
+      "mean_token_accuracy": 0.42437282159924505,
+      "num_tokens": 1684441.0,
+      "step": 550
+    },
+    {
+      "entropy": 2.703564453125,
+      "epoch": 0.04,
+      "grad_norm": 2.303372383117676,
+      "learning_rate": 9.997412687745474e-05,
+      "loss": 2.8048,
+      "mean_token_accuracy": 0.42454416677355766,
+      "num_tokens": 1838012.0,
+      "step": 600
+    },
+    {
+      "entropy": 2.697548828125,
+      "epoch": 0.043333333333333335,
+      "grad_norm": 1.6631819009780884,
+      "learning_rate": 9.995385197674353e-05,
+      "loss": 2.7978,
+      "mean_token_accuracy": 0.4260334567725658,
+      "num_tokens": 1991026.0,
+      "step": 650
+    },
+    {
+      "entropy": 2.737724609375,
+      "epoch": 0.04666666666666667,
+      "grad_norm": 1.8547998666763306,
+      "learning_rate": 9.992775499116298e-05,
+      "loss": 2.8385,
+      "mean_token_accuracy": 0.4228105416893959,
+      "num_tokens": 2143574.0,
+      "step": 700
+    },
+    {
+      "entropy": 2.723037109375,
+      "epoch": 0.05,
+      "grad_norm": 2.5471417903900146,
+      "learning_rate": 9.989583896229766e-05,
+      "loss": 2.8284,
+      "mean_token_accuracy": 0.42690352082252503,
+      "num_tokens": 2296869.0,
+      "step": 750
+    },
+    {
+      "entropy": 2.73912109375,
+      "epoch": 0.05333333333333334,
+      "grad_norm": 1.8039970397949219,
+      "learning_rate": 9.985810760993735e-05,
+      "loss": 2.8501,
+      "mean_token_accuracy": 0.4239633755385876,
+      "num_tokens": 2448287.0,
+      "step": 800
+    },
+    {
+      "entropy": 2.725390625,
+      "epoch": 0.056666666666666664,
+      "grad_norm": 4.132200717926025,
+      "learning_rate": 9.981456533164355e-05,
+      "loss": 2.8181,
+      "mean_token_accuracy": 0.42553551197052003,
+      "num_tokens": 2602725.0,
+      "step": 850
+    },
+    {
+      "entropy": 2.693154296875,
+      "epoch": 0.06,
+      "grad_norm": 1.8514758348464966,
+      "learning_rate": 9.97652172022369e-05,
+      "loss": 2.7929,
+      "mean_token_accuracy": 0.4280212070047855,
+      "num_tokens": 2756488.0,
+      "step": 900
+    },
+    {
+      "entropy": 2.6425732421875,
+      "epoch": 0.06333333333333334,
+      "grad_norm": 2.878779172897339,
+      "learning_rate": 9.971006897320579e-05,
+      "loss": 2.7221,
+      "mean_token_accuracy": 0.44406644865870476,
+      "num_tokens": 2911514.0,
+      "step": 950
+    },
+    {
+      "entropy": 2.66560546875,
+      "epoch": 0.06666666666666667,
+      "grad_norm": 3.8290674686431885,
+      "learning_rate": 9.964912707203586e-05,
+      "loss": 2.7872,
+      "mean_token_accuracy": 0.43023131355643274,
+      "num_tokens": 3064010.0,
+      "step": 1000
+    },
+    {
+      "entropy": 2.68474609375,
+      "epoch": 0.07,
+      "grad_norm": 2.2297792434692383,
+      "learning_rate": 9.958239860146112e-05,
+      "loss": 2.7963,
+      "mean_token_accuracy": 0.4303510873019695,
+      "num_tokens": 3216525.0,
+      "step": 1050
+    },
+    {
+      "entropy": 2.6753564453125,
+      "epoch": 0.07333333333333333,
+      "grad_norm": 2.2983882427215576,
+      "learning_rate": 9.950989133863593e-05,
+      "loss": 2.7701,
+      "mean_token_accuracy": 0.4338577099144459,
+      "num_tokens": 3370033.0,
+      "step": 1100
+    },
+    {
+      "entropy": 2.759462890625,
+      "epoch": 0.07666666666666666,
+      "grad_norm": 1.61794114112854,
+      "learning_rate": 9.943161373422868e-05,
+      "loss": 2.8555,
+      "mean_token_accuracy": 0.4244931024312973,
+      "num_tokens": 3523171.0,
+      "step": 1150
+    },
+    {
+      "entropy": 2.46953857421875,
+      "epoch": 0.08,
+      "grad_norm": 1.5758808851242065,
+      "learning_rate": 9.934757491143687e-05,
+      "loss": 2.5736,
+      "mean_token_accuracy": 0.47368608459830286,
+      "num_tokens": 3677091.0,
+      "step": 1200
+    },
+    {
+      "entropy": 2.226496276855469,
+      "epoch": 0.08333333333333333,
+      "grad_norm": 4.1404852867126465,
+      "learning_rate": 9.925778466492372e-05,
+      "loss": 2.3526,
+      "mean_token_accuracy": 0.5196963116526604,
+      "num_tokens": 3836663.0,
+      "step": 1250
+    },
+    {
+      "entropy": 2.764296875,
+      "epoch": 0.08666666666666667,
+      "grad_norm": 1.473595380783081,
+      "learning_rate": 9.916225345967676e-05,
+      "loss": 2.8245,
+      "mean_token_accuracy": 0.424507200717926,
+      "num_tokens": 3989426.0,
+      "step": 1300
+    },
+    {
+      "entropy": 2.717734375,
+      "epoch": 0.09,
+      "grad_norm": 3.0632381439208984,
+      "learning_rate": 9.906099242978796e-05,
+      "loss": 2.8139,
+      "mean_token_accuracy": 0.42621336653828623,
+      "num_tokens": 4142400.0,
+      "step": 1350
+    },
+    {
+      "entropy": 2.75982421875,
+      "epoch": 0.09333333333333334,
+      "grad_norm": 1.513647437095642,
+      "learning_rate": 9.895401337715624e-05,
+      "loss": 2.8607,
+      "mean_token_accuracy": 0.4194491642713547,
+      "num_tokens": 4295903.0,
+      "step": 1400
+    },
+    {
+      "entropy": 2.6927099609375,
+      "epoch": 0.09666666666666666,
+      "grad_norm": 1.5781805515289307,
+      "learning_rate": 9.884132877011183e-05,
+      "loss": 2.7784,
+      "mean_token_accuracy": 0.43057438001036646,
+      "num_tokens": 4450117.0,
+      "step": 1450
+    },
+    {
+      "entropy": 2.7134375,
+      "epoch": 0.1,
+      "grad_norm": 1.941235065460205,
+      "learning_rate": 9.872295174196317e-05,
+      "loss": 2.7922,
+      "mean_token_accuracy": 0.4271659015119076,
+      "num_tokens": 4603551.0,
+      "step": 1500
+    },
+    {
+      "entropy": 2.708359375,
+      "epoch": 0.10333333333333333,
+      "grad_norm": 3.3001255989074707,
+      "learning_rate": 9.859889608946621e-05,
+      "loss": 2.7784,
+      "mean_token_accuracy": 0.4277858440577984,
+      "num_tokens": 4757625.0,
+      "step": 1550
+    },
+    {
+      "entropy": 2.7323974609375,
+      "epoch": 0.10666666666666667,
+      "grad_norm": 1.6675961017608643,
+      "learning_rate": 9.846917627121644e-05,
+      "loss": 2.8315,
+      "mean_token_accuracy": 0.42472330912947653,
+      "num_tokens": 4911005.0,
+      "step": 1600
+    },
+    {
+      "entropy": 2.755791015625,
+      "epoch": 0.11,
+      "grad_norm": 1.9695419073104858,
+      "learning_rate": 9.833380740596366e-05,
+      "loss": 2.8338,
+      "mean_token_accuracy": 0.4245320174098015,
+      "num_tokens": 5063471.0,
+      "step": 1650
+    },
+    {
+      "entropy": 2.762841796875,
+      "epoch": 0.11333333333333333,
+      "grad_norm": 1.7147846221923828,
+      "learning_rate": 9.819280527084997e-05,
+      "loss": 2.8418,
+      "mean_token_accuracy": 0.42281755566596985,
+      "num_tokens": 5216120.0,
+      "step": 1700
+    },
+    {
+      "entropy": 2.780341796875,
+      "epoch": 0.11666666666666667,
+      "grad_norm": 2.6947574615478516,
+      "learning_rate": 9.8046186299571e-05,
+      "loss": 2.8742,
+      "mean_token_accuracy": 0.42031392723321914,
+      "num_tokens": 5371264.0,
+      "step": 1750
+    },
+    {
+      "entropy": 2.785537109375,
+      "epoch": 0.12,
+      "grad_norm": 1.9052940607070923,
+      "learning_rate": 9.78939675804605e-05,
+      "loss": 2.8828,
+      "mean_token_accuracy": 0.4183352366089821,
+      "num_tokens": 5523867.0,
+      "step": 1800
+    },
+    {
+      "entropy": 2.760068359375,
+      "epoch": 0.12333333333333334,
+      "grad_norm": 1.9224604368209839,
+      "learning_rate": 9.773616685449871e-05,
+      "loss": 2.8583,
+      "mean_token_accuracy": 0.4213790901005268,
+      "num_tokens": 5677161.0,
+      "step": 1850
+    },
+    {
+      "entropy": 2.723115234375,
+      "epoch": 0.12666666666666668,
+      "grad_norm": 6.709502696990967,
+      "learning_rate": 9.757280251324468e-05,
+      "loss": 2.7988,
+      "mean_token_accuracy": 0.42887720301747323,
+      "num_tokens": 5830674.0,
+      "step": 1900
+    },
+    {
+      "entropy": 2.7593603515625,
+      "epoch": 0.13,
+      "grad_norm": 1.7702646255493164,
+      "learning_rate": 9.740389359669278e-05,
+      "loss": 2.8448,
+      "mean_token_accuracy": 0.42266563907265664,
+      "num_tokens": 5983575.0,
+      "step": 1950
+    },
+    {
+      "entropy": 2.7628125,
+      "epoch": 0.13333333333333333,
+      "grad_norm": 2.2006354331970215,
+      "learning_rate": 9.722945979105351e-05,
+      "loss": 2.8523,
+      "mean_token_accuracy": 0.4193016523122787,
+      "num_tokens": 6137274.0,
+      "step": 2000
+    },
+    {
+      "entropy": 2.709619140625,
+      "epoch": 0.13666666666666666,
+      "grad_norm": 2.978189468383789,
+      "learning_rate": 9.704952142645925e-05,
+      "loss": 2.7821,
+      "mean_token_accuracy": 0.4332714307308197,
+      "num_tokens": 6290403.0,
+      "step": 2050
+    },
+    {
+      "entropy": 2.786767578125,
+      "epoch": 0.14,
+      "grad_norm": 1.835463047027588,
+      "learning_rate": 9.686409947459458e-05,
+      "loss": 2.8772,
+      "mean_token_accuracy": 0.41907066956162453,
+      "num_tokens": 6443472.0,
+      "step": 2100
+    },
+    {
+      "entropy": 2.765498046875,
+      "epoch": 0.14333333333333334,
+      "grad_norm": 1.9685204029083252,
+      "learning_rate": 9.667321554625225e-05,
+      "loss": 2.8375,
+      "mean_token_accuracy": 0.4219497123360634,
+      "num_tokens": 6597285.0,
+      "step": 2150
+    },
+    {
+      "entropy": 2.788427734375,
+      "epoch": 0.14666666666666667,
+      "grad_norm": 1.5145137310028076,
+      "learning_rate": 9.647689188881431e-05,
+      "loss": 2.8716,
+      "mean_token_accuracy": 0.41897833287715913,
+      "num_tokens": 6750742.0,
+      "step": 2200
+    },
+    {
+      "entropy": 2.642529296875,
+      "epoch": 0.15,
+      "grad_norm": 1.770254373550415,
+      "learning_rate": 9.62751513836593e-05,
+      "loss": 2.7125,
+      "mean_token_accuracy": 0.43980305790901186,
+      "num_tokens": 6903600.0,
+      "step": 2250
+    },
+    {
+      "entropy": 2.7381689453125,
+      "epoch": 0.15333333333333332,
+      "grad_norm": 1.5834028720855713,
+      "learning_rate": 9.606801754349536e-05,
+      "loss": 2.8267,
+      "mean_token_accuracy": 0.42276035860180855,
+      "num_tokens": 7056618.0,
+      "step": 2300
+    },
+    {
+      "entropy": 2.6690234375,
+      "epoch": 0.15666666666666668,
+      "grad_norm": 2.3597967624664307,
+      "learning_rate": 9.58555145096199e-05,
+      "loss": 2.7116,
+      "mean_token_accuracy": 0.44172772198915483,
+      "num_tokens": 7211139.0,
+      "step": 2350
+    },
+    {
+      "entropy": 2.768017578125,
+      "epoch": 0.16,
+      "grad_norm": 1.9302066564559937,
+      "learning_rate": 9.563766704910587e-05,
+      "loss": 2.8864,
+      "mean_token_accuracy": 0.41909650415182115,
+      "num_tokens": 7364151.0,
+      "step": 2400
+    },
+    {
+      "entropy": 2.780185546875,
+      "epoch": 0.16333333333333333,
+      "grad_norm": 1.3915791511535645,
+      "learning_rate": 9.541450055191532e-05,
+      "loss": 2.8651,
+      "mean_token_accuracy": 0.42147023320198057,
+      "num_tokens": 7517656.0,
+      "step": 2450
+    },
+    {
+      "entropy": 2.5751953125,
+      "epoch": 0.16666666666666666,
+      "grad_norm": 1.6701171398162842,
+      "learning_rate": 9.518604102794009e-05,
+      "loss": 2.64,
+      "mean_token_accuracy": 0.4603299044072628,
+      "num_tokens": 7673602.0,
+      "step": 2500
+    },
+    {
+      "entropy": 2.818955078125,
+      "epoch": 0.17,
+      "grad_norm": 1.7646194696426392,
+      "learning_rate": 9.495231510397038e-05,
+      "loss": 2.8998,
+      "mean_token_accuracy": 0.4178720970451832,
+      "num_tokens": 7826563.0,
+      "step": 2550
+    },
+    {
+      "entropy": 2.7537109375,
+      "epoch": 0.17333333333333334,
+      "grad_norm": 1.6351436376571655,
+      "learning_rate": 9.471335002059149e-05,
+      "loss": 2.8501,
+      "mean_token_accuracy": 0.42110710859298706,
+      "num_tokens": 7980653.0,
+      "step": 2600
+    },
+    {
+      "entropy": 2.817265625,
+      "epoch": 0.17666666666666667,
+      "grad_norm": 1.7285945415496826,
+      "learning_rate": 9.446917362900891e-05,
+      "loss": 2.9076,
+      "mean_token_accuracy": 0.4160151961445808,
+      "num_tokens": 8133185.0,
+      "step": 2650
+    },
+    {
+      "entropy": 2.7827685546875,
+      "epoch": 0.18,
+      "grad_norm": 1.4033441543579102,
+      "learning_rate": 9.42198143878023e-05,
+      "loss": 2.8586,
+      "mean_token_accuracy": 0.42112465128302573,
+      "num_tokens": 8285809.0,
+      "step": 2700
+    },
+    {
+      "entropy": 2.774326171875,
+      "epoch": 0.18333333333333332,
+      "grad_norm": 1.9107087850570679,
+      "learning_rate": 9.39653013596086e-05,
+      "loss": 2.8601,
+      "mean_token_accuracy": 0.42484147623181345,
+      "num_tokens": 8437014.0,
+      "step": 2750
+    },
+    {
+      "entropy": 2.7293310546875,
+      "epoch": 0.18666666666666668,
+      "grad_norm": 1.7647576332092285,
+      "learning_rate": 9.37056642077349e-05,
+      "loss": 2.8109,
+      "mean_token_accuracy": 0.42852689415216444,
+      "num_tokens": 8590841.0,
+      "step": 2800
+    },
+    {
+      "entropy": 2.784404296875,
+      "epoch": 0.19,
+      "grad_norm": 1.812900424003601,
+      "learning_rate": 9.344093319270112e-05,
+      "loss": 2.8602,
+      "mean_token_accuracy": 0.4225932811200619,
+      "num_tokens": 8744061.0,
+      "step": 2850
+    },
+    {
+      "entropy": 2.75974609375,
+      "epoch": 0.19333333333333333,
+      "grad_norm": 1.953025460243225,
+      "learning_rate": 9.317113916871322e-05,
+      "loss": 2.8448,
+      "mean_token_accuracy": 0.42387092173099516,
+      "num_tokens": 8898279.0,
+      "step": 2900
+    },
+    {
+      "entropy": 2.763447265625,
+      "epoch": 0.19666666666666666,
+      "grad_norm": 3.064847707748413,
+      "learning_rate": 9.289631358006715e-05,
+      "loss": 2.8537,
+      "mean_token_accuracy": 0.42707418724894525,
+      "num_tokens": 9054284.0,
+      "step": 2950
+    },
+    {
+      "entropy": 2.806103515625,
+      "epoch": 0.2,
+      "grad_norm": 1.4979510307312012,
+      "learning_rate": 9.261648845748401e-05,
+      "loss": 2.8828,
+      "mean_token_accuracy": 0.4184405809640884,
+      "num_tokens": 9207157.0,
+      "step": 3000
+    },
+    {
+      "entropy": 2.6966796875,
+      "epoch": 0.20333333333333334,
+      "grad_norm": 1.7699885368347168,
+      "learning_rate": 9.233169641437694e-05,
+      "loss": 2.7583,
+      "mean_token_accuracy": 0.4364832893013954,
+      "num_tokens": 9361357.0,
+      "step": 3050
+    },
+    {
+      "entropy": 2.7894384765625,
+      "epoch": 0.20666666666666667,
+      "grad_norm": 1.7575098276138306,
+      "learning_rate": 9.204197064305001e-05,
+      "loss": 2.8602,
+      "mean_token_accuracy": 0.42046931490302086,
+      "num_tokens": 9513949.0,
+      "step": 3100
+    },
+    {
+      "entropy": 2.7716552734375,
+      "epoch": 0.21,
+      "grad_norm": 1.4345855712890625,
+      "learning_rate": 9.174734491082976e-05,
+      "loss": 2.8573,
+      "mean_token_accuracy": 0.42368178591132166,
+      "num_tokens": 9667494.0,
+      "step": 3150
+    },
+    {
+      "entropy": 2.760703125,
+      "epoch": 0.21333333333333335,
+      "grad_norm": 1.9951531887054443,
+      "learning_rate": 9.14478535561295e-05,
+      "loss": 2.8315,
+      "mean_token_accuracy": 0.4247917515039444,
+      "num_tokens": 9820692.0,
+      "step": 3200
+    },
+    {
+      "entropy": 2.781962890625,
+      "epoch": 0.21666666666666667,
+      "grad_norm": 2.224896192550659,
+      "learning_rate": 9.114353148444735e-05,
+      "loss": 2.8542,
+      "mean_token_accuracy": 0.42145723387598993,
+      "num_tokens": 9973866.0,
+      "step": 3250
+    },
+    {
+      "entropy": 2.78275390625,
+      "epoch": 0.22,
+      "grad_norm": 1.8612408638000488,
+      "learning_rate": 9.08344141642979e-05,
+      "loss": 2.8495,
+      "mean_token_accuracy": 0.4234173122048378,
+      "num_tokens": 10127001.0,
+      "step": 3300
+    },
+    {
+      "entropy": 2.79578125,
+      "epoch": 0.22333333333333333,
+      "grad_norm": 1.4872896671295166,
+      "learning_rate": 9.052053762307848e-05,
+      "loss": 2.871,
+      "mean_token_accuracy": 0.41822456374764444,
+      "num_tokens": 10279434.0,
+      "step": 3350
+    },
+    {
+      "entropy": 2.8016796875,
+      "epoch": 0.22666666666666666,
+      "grad_norm": 1.5314096212387085,
+      "learning_rate": 9.020193844287012e-05,
+      "loss": 2.881,
+      "mean_token_accuracy": 0.41675362810492517,
+      "num_tokens": 10432221.0,
+      "step": 3400
+    },
+    {
+      "entropy": 2.8003515625,
+      "epoch": 0.23,
+      "grad_norm": 1.9040875434875488,
+      "learning_rate": 8.987865375617402e-05,
+      "loss": 2.8765,
+      "mean_token_accuracy": 0.42052937611937524,
+      "num_tokens": 10585539.0,
+      "step": 3450
+    },
+    {
+      "entropy": 2.783642578125,
+      "epoch": 0.23333333333333334,
+      "grad_norm": 3.2793874740600586,
+      "learning_rate": 8.955072124158362e-05,
+      "loss": 2.8473,
+      "mean_token_accuracy": 0.42251690953969956,
+      "num_tokens": 10740811.0,
+      "step": 3500
+    },
+    {
+      "entropy": 2.758759765625,
+      "epoch": 0.23666666666666666,
+      "grad_norm": 1.7231287956237793,
+      "learning_rate": 8.921817911939343e-05,
+      "loss": 2.8492,
+      "mean_token_accuracy": 0.42849296987056734,
+      "num_tokens": 10894631.0,
+      "step": 3550
+    },
+    {
+      "entropy": 2.785625,
+      "epoch": 0.24,
+      "grad_norm": 1.48932683467865,
+      "learning_rate": 8.888106614714424e-05,
+      "loss": 2.8575,
+      "mean_token_accuracy": 0.4251786956191063,
+      "num_tokens": 11048023.0,
+      "step": 3600
+    },
+    {
+      "entropy": 2.729296875,
+      "epoch": 0.24333333333333335,
+      "grad_norm": 3.8472039699554443,
+      "learning_rate": 8.853942161510616e-05,
+      "loss": 2.7997,
+      "mean_token_accuracy": 0.4357279340922833,
+      "num_tokens": 11202143.0,
+      "step": 3650
+    },
+    {
+      "entropy": 2.7321875,
+      "epoch": 0.24666666666666667,
+      "grad_norm": 1.7587218284606934,
+      "learning_rate": 8.819328534169922e-05,
+      "loss": 2.8042,
+      "mean_token_accuracy": 0.42584114834666253,
+      "num_tokens": 11356394.0,
+      "step": 3700
+    },
+    {
+      "entropy": 2.8251953125,
+      "epoch": 0.25,
+      "grad_norm": 2.1818931102752686,
+      "learning_rate": 8.78426976688526e-05,
+      "loss": 2.8964,
+      "mean_token_accuracy": 0.4179812644422054,
+      "num_tokens": 11509121.0,
+      "step": 3750
+    },
+    {
+      "entropy": 2.789833984375,
+      "epoch": 0.25333333333333335,
+      "grad_norm": 2.112947463989258,
+      "learning_rate": 8.748769945730288e-05,
+      "loss": 2.8656,
+      "mean_token_accuracy": 0.4251732324063778,
+      "num_tokens": 11662329.0,
+      "step": 3800
+    },
+    {
+      "entropy": 2.792666015625,
+      "epoch": 0.25666666666666665,
+      "grad_norm": 1.513971209526062,
+      "learning_rate": 8.712833208183166e-05,
+      "loss": 2.8566,
+      "mean_token_accuracy": 0.42212075248360637,
+      "num_tokens": 11815604.0,
+      "step": 3850
+    },
+    {
+      "entropy": 2.785654296875,
+      "epoch": 0.26,
+      "grad_norm": 3.849529981613159,
+      "learning_rate": 8.676463742644341e-05,
+      "loss": 2.8656,
+      "mean_token_accuracy": 0.422549641430378,
+      "num_tokens": 11968459.0,
+      "step": 3900
+    },
+    {
+      "entropy": 2.757626953125,
+      "epoch": 0.2633333333333333,
+      "grad_norm": 2.1418285369873047,
+      "learning_rate": 8.639665787948391e-05,
+      "loss": 2.8146,
+      "mean_token_accuracy": 0.4283721697330475,
+      "num_tokens": 12122211.0,
+      "step": 3950
+    },
+    {
+      "entropy": 2.795537109375,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.8683708906173706,
+      "learning_rate": 8.602443632869988e-05,
+      "loss": 2.8787,
+      "mean_token_accuracy": 0.41945122107863425,
+      "num_tokens": 12275938.0,
+      "step": 4000
+    },
+    {
+      "entropy": 2.8119921875,
+      "epoch": 0.27,
+      "grad_norm": 1.687473177909851,
+      "learning_rate": 8.564801615624053e-05,
+      "loss": 2.8852,
+      "mean_token_accuracy": 0.4206313456594944,
+      "num_tokens": 12429202.0,
+      "step": 4050
+    },
+    {
+      "entropy": 2.773330078125,
+      "epoch": 0.2733333333333333,
+      "grad_norm": 1.4739123582839966,
+      "learning_rate": 8.526744123360133e-05,
+      "loss": 2.8549,
+      "mean_token_accuracy": 0.42225955575704577,
+      "num_tokens": 12582688.0,
+      "step": 4100
+    },
+    {
+      "entropy": 2.7910546875,
+      "epoch": 0.27666666666666667,
+      "grad_norm": 1.9559763669967651,
+      "learning_rate": 8.488275591651081e-05,
+      "loss": 2.8615,
+      "mean_token_accuracy": 0.4222630296647549,
+      "num_tokens": 12736626.0,
+      "step": 4150
+    },
+    {
+      "entropy": 2.821591796875,
+      "epoch": 0.28,
+      "grad_norm": 1.7581783533096313,
+      "learning_rate": 8.449400503976103e-05,
+      "loss": 2.8913,
+      "mean_token_accuracy": 0.4204542309045792,
+      "num_tokens": 12888872.0,
+      "step": 4200
+    },
+    {
+      "entropy": 2.800439453125,
+      "epoch": 0.2833333333333333,
+      "grad_norm": 1.909140944480896,
+      "learning_rate": 8.410123391198197e-05,
+      "loss": 2.8761,
+      "mean_token_accuracy": 0.4198679532110691,
+      "num_tokens": 13041769.0,
+      "step": 4250
+    },
+    {
+      "entropy": 2.785537109375,
+      "epoch": 0.2866666666666667,
+      "grad_norm": 2.516155481338501,
+      "learning_rate": 8.3704488310361e-05,
+      "loss": 2.8549,
+      "mean_token_accuracy": 0.4194681803882122,
+      "num_tokens": 153377.0,
+      "step": 4300
+    },
+    {
+      "entropy": 2.823359375,
+      "epoch": 0.29,
+      "grad_norm": 1.5486787557601929,
+      "learning_rate": 8.330381447530747e-05,
+      "loss": 2.9057,
+      "mean_token_accuracy": 0.41606380403041837,
+      "num_tokens": 306041.0,
+      "step": 4350
+    },
+    {
+      "entropy": 2.7942578125,
+      "epoch": 0.29333333333333333,
+      "grad_norm": 1.7242319583892822,
+      "learning_rate": 8.289925910506346e-05,
+      "loss": 2.8545,
+      "mean_token_accuracy": 0.42336358562111853,
+      "num_tokens": 459047.0,
+      "step": 4400
+    },
+    {
+      "entropy": 2.813759765625,
+      "epoch": 0.2966666666666667,
+      "grad_norm": 4.094738960266113,
+      "learning_rate": 8.249086935026114e-05,
+      "loss": 2.8827,
+      "mean_token_accuracy": 0.4195826704800129,
+      "num_tokens": 612962.0,
+      "step": 4450
+    },
+    {
+      "entropy": 2.77751953125,
+      "epoch": 0.3,
+      "grad_norm": 1.7947832345962524,
+      "learning_rate": 8.207869280842738e-05,
+      "loss": 2.8503,
+      "mean_token_accuracy": 0.4240879508852959,
+      "num_tokens": 767235.0,
+      "step": 4500
+    },
+    {
+      "entropy": 2.79923828125,
+      "epoch": 0.30333333333333334,
+      "grad_norm": 2.434159517288208,
+      "learning_rate": 8.166277751843632e-05,
+      "loss": 2.8675,
+      "mean_token_accuracy": 0.4212272399663925,
+      "num_tokens": 154786.0,
+      "step": 4550
+    },
+    {
+      "entropy": 2.763828125,
+      "epoch": 0.30666666666666664,
+      "grad_norm": 3.8289670944213867,
+      "learning_rate": 8.124317195491044e-05,
+      "loss": 2.8254,
+      "mean_token_accuracy": 0.4252131301164627,
+      "num_tokens": 308538.0,
+      "step": 4600
+    },
+    {
+      "entropy": 2.7729248046875,
+      "epoch": 0.31,
+      "grad_norm": 2.0406832695007324,
+      "learning_rate": 8.081992502257095e-05,
+      "loss": 2.8387,
+      "mean_token_accuracy": 0.42438063144683835,
+      "num_tokens": 461672.0,
+      "step": 4650
+    },
+    {
+      "entropy": 2.8070703125,
+      "epoch": 0.31333333333333335,
+      "grad_norm": 1.755568265914917,
+      "learning_rate": 8.039308605053788e-05,
+      "loss": 2.8856,
+      "mean_token_accuracy": 0.4205561250448227,
+      "num_tokens": 615569.0,
+      "step": 4700
+    },
+    {
+      "entropy": 2.77701171875,
+      "epoch": 0.31666666666666665,
+      "grad_norm": 1.8182449340820312,
+      "learning_rate": 7.996270478658084e-05,
+      "loss": 2.8534,
+      "mean_token_accuracy": 0.4222332505881786,
+      "num_tokens": 768145.0,
+      "step": 4750
+    },
+    {
+      "entropy": 2.821943359375,
+      "epoch": 0.32,
+      "grad_norm": 2.8805055618286133,
+      "learning_rate": 7.952883139132104e-05,
+      "loss": 2.8929,
+      "mean_token_accuracy": 0.4197997708618641,
+      "num_tokens": 921607.0,
+      "step": 4800
+    },
+    {
+      "entropy": 2.7778515625,
+      "epoch": 0.3233333333333333,
+      "grad_norm": 1.652905821800232,
+      "learning_rate": 7.909151643238497e-05,
+      "loss": 2.8585,
+      "mean_token_accuracy": 0.41916815549135206,
+      "num_tokens": 1075100.0,
+      "step": 4850
+    },
+    {
+      "entropy": 2.784873046875,
+      "epoch": 0.32666666666666666,
+      "grad_norm": 2.501504421234131,
+      "learning_rate": 7.865081087851088e-05,
+      "loss": 2.8601,
+      "mean_token_accuracy": 0.42385279685258864,
+      "num_tokens": 1229298.0,
+      "step": 4900
+    },
+    {
+      "entropy": 2.826240234375,
+      "epoch": 0.33,
+      "grad_norm": 2.0058343410491943,
+      "learning_rate": 7.820676609360838e-05,
+      "loss": 2.9167,
+      "mean_token_accuracy": 0.4164962163567543,
+      "num_tokens": 1382332.0,
+      "step": 4950
+    },
+    {
+      "entropy": 2.8257421875,
+      "epoch": 0.3333333333333333,
+      "grad_norm": 2.2245631217956543,
+      "learning_rate": 7.775943383077195e-05,
+      "loss": 2.8978,
+      "mean_token_accuracy": 0.41643302455544473,
+      "num_tokens": 1535929.0,
+      "step": 5000
+    },
+    {
+      "entropy": 2.791455078125,
+      "epoch": 0.33666666666666667,
+      "grad_norm": 4.734055519104004,
+      "learning_rate": 7.730886622624934e-05,
+      "loss": 2.8511,
+      "mean_token_accuracy": 0.42518281653523443,
+      "num_tokens": 1689131.0,
+      "step": 5050
+    },
+    {
+      "entropy": 2.818017578125,
+      "epoch": 0.34,
+      "grad_norm": 1.875542402267456,
+      "learning_rate": 7.685511579336488e-05,
+      "loss": 2.8885,
+      "mean_token_accuracy": 0.41853062853217127,
+      "num_tokens": 1842796.0,
+      "step": 5100
+    },
+    {
+      "entropy": 2.827646484375,
+      "epoch": 0.3433333333333333,
+      "grad_norm": 4.402409076690674,
+      "learning_rate": 7.639823541639927e-05,
+      "loss": 2.905,
+      "mean_token_accuracy": 0.4157783268392086,
+      "num_tokens": 1995968.0,
+      "step": 5150
+    },
+    {
+      "entropy": 2.84697265625,
+      "epoch": 0.3466666666666667,
+      "grad_norm": 2.0653014183044434,
+      "learning_rate": 7.593827834442592e-05,
+      "loss": 2.9107,
+      "mean_token_accuracy": 0.41715434193611145,
+      "num_tokens": 2150177.0,
+      "step": 5200
+    },
+    {
+      "entropy": 2.760859375,
+      "epoch": 0.35,
+      "grad_norm": 2.4690027236938477,
+      "learning_rate": 7.547529818510475e-05,
+      "loss": 2.8484,
+      "mean_token_accuracy": 0.42122889682650566,
+      "num_tokens": 2303767.0,
+      "step": 5250
+    },
+    {
+      "entropy": 2.799462890625,
+      "epoch": 0.35333333333333333,
+      "grad_norm": 1.8778904676437378,
+      "learning_rate": 7.50093488984343e-05,
+      "loss": 2.8664,
+      "mean_token_accuracy": 0.4221074862778187,
+      "num_tokens": 2456365.0,
+      "step": 5300
+    },
+    {
+      "entropy": 2.812568359375,
+      "epoch": 0.3566666666666667,
+      "grad_norm": 2.1682090759277344,
+      "learning_rate": 7.454048479046277e-05,
+      "loss": 2.8881,
+      "mean_token_accuracy": 0.4154551622271538,
+      "num_tokens": 2609470.0,
+      "step": 5350
+    },
+    {
+      "entropy": 2.8146337890625,
+      "epoch": 0.36,
+      "grad_norm": 2.3601555824279785,
+      "learning_rate": 7.406876050695859e-05,
+      "loss": 2.8836,
+      "mean_token_accuracy": 0.4197733694314957,
+      "num_tokens": 2763581.0,
+      "step": 5400
+    },
+    {
+      "entropy": 2.773623046875,
+      "epoch": 0.36333333333333334,
+      "grad_norm": 1.702940583229065,
+      "learning_rate": 7.359423102704161e-05,
+      "loss": 2.8451,
+      "mean_token_accuracy": 0.42223391726613047,
+      "num_tokens": 2916751.0,
+      "step": 5450
+    },
+    {
+      "entropy": 2.83443359375,
+      "epoch": 0.36666666666666664,
+      "grad_norm": 2.1349689960479736,
+      "learning_rate": 7.311695165677523e-05,
+      "loss": 2.9013,
+      "mean_token_accuracy": 0.41880217954516413,
+      "num_tokens": 3070227.0,
+      "step": 5500
+    },
+    {
+      "entropy": 2.82658203125,
+      "epoch": 0.37,
+      "grad_norm": 4.791004657745361,
+      "learning_rate": 7.263697802272056e-05,
+      "loss": 2.9068,
+      "mean_token_accuracy": 0.41114905059337614,
+      "num_tokens": 3223643.0,
+      "step": 5550
+    },
+    {
+      "entropy": 2.801572265625,
+      "epoch": 0.37333333333333335,
+      "grad_norm": 3.701864719390869,
+      "learning_rate": 7.215436606545323e-05,
+      "loss": 2.8767,
+      "mean_token_accuracy": 0.4190987147390842,
+      "num_tokens": 3377864.0,
+      "step": 5600
+    },
+    {
+      "entropy": 2.75138671875,
+      "epoch": 0.37666666666666665,
+      "grad_norm": 1.8020061254501343,
+      "learning_rate": 7.16691720330434e-05,
+      "loss": 2.8241,
+      "mean_token_accuracy": 0.4296322312951088,
+      "num_tokens": 3531566.0,
+      "step": 5650
+    },
+    {
+      "entropy": 2.8160107421875,
+      "epoch": 0.38,
+      "grad_norm": 2.0307564735412598,
+      "learning_rate": 7.118145247450033e-05,
+      "loss": 2.8915,
+      "mean_token_accuracy": 0.42226585134863853,
+      "num_tokens": 3685105.0,
+      "step": 5700
+    },
+    {
+      "entropy": 2.8155859375,
+      "epoch": 0.38333333333333336,
+      "grad_norm": 2.144331693649292,
+      "learning_rate": 7.06912642331814e-05,
+      "loss": 2.8781,
+      "mean_token_accuracy": 0.418236046731472,
+      "num_tokens": 3838999.0,
+      "step": 5750
+    },
+    {
+      "entropy": 2.80875,
+      "epoch": 0.38666666666666666,
+      "grad_norm": 2.043344497680664,
+      "learning_rate": 7.01986644401672e-05,
+      "loss": 2.8866,
+      "mean_token_accuracy": 0.4175939907133579,
+      "num_tokens": 3992733.0,
+      "step": 5800
+    },
+    {
+      "entropy": 2.81671875,
+      "epoch": 0.39,
+      "grad_norm": 1.998583436012268,
+      "learning_rate": 6.970371050760291e-05,
+      "loss": 2.8791,
+      "mean_token_accuracy": 0.4197034777700901,
+      "num_tokens": 4146003.0,
+      "step": 5850
+    },
+    {
+      "entropy": 2.8267578125,
+      "epoch": 0.3933333333333333,
+      "grad_norm": 2.0911405086517334,
+      "learning_rate": 6.920646012200693e-05,
+      "loss": 2.9084,
+      "mean_token_accuracy": 0.41816318228840826,
+      "num_tokens": 4298968.0,
+      "step": 5900
+    },
+    {
+      "entropy": 2.79427734375,
+      "epoch": 0.39666666666666667,
+      "grad_norm": 3.2191355228424072,
+      "learning_rate": 6.870697123754762e-05,
+      "loss": 2.8667,
+      "mean_token_accuracy": 0.4213761845231056,
+      "num_tokens": 4452607.0,
+      "step": 5950
+    },
+    {
+      "entropy": 2.8197265625,
+      "epoch": 0.4,
+      "grad_norm": 2.1716251373291016,
+      "learning_rate": 6.820530206928869e-05,
+      "loss": 2.8931,
+      "mean_token_accuracy": 0.41745673775672915,
+      "num_tokens": 4606820.0,
+      "step": 6000
+    },
+    {
+      "entropy": 2.85451171875,
+      "epoch": 0.4033333333333333,
+      "grad_norm": 1.6433621644973755,
+      "learning_rate": 6.770151108640437e-05,
+      "loss": 2.9328,
+      "mean_token_accuracy": 0.41258298978209496,
+      "num_tokens": 4759613.0,
+      "step": 6050
+    },
+    {
+      "entropy": 2.8144921875,
+      "epoch": 0.4066666666666667,
+      "grad_norm": 3.3031110763549805,
+      "learning_rate": 6.719565700536478e-05,
+      "loss": 2.8946,
+      "mean_token_accuracy": 0.41846631169319154,
+      "num_tokens": 4912212.0,
+      "step": 6100
+    },
+    {
+      "entropy": 2.829951171875,
+      "epoch": 0.41,
+      "grad_norm": 2.026379108428955,
+      "learning_rate": 6.66877987830927e-05,
+      "loss": 2.9061,
+      "mean_token_accuracy": 0.4161474916338921,
+      "num_tokens": 5065446.0,
+      "step": 6150
+    },
+    {
+      "entropy": 2.817451171875,
+      "epoch": 0.41333333333333333,
+      "grad_norm": 2.2108099460601807,
+      "learning_rate": 6.617799561009204e-05,
+      "loss": 2.8878,
+      "mean_token_accuracy": 0.41852827087044714,
+      "num_tokens": 5218508.0,
+      "step": 6200
+    },
+    {
+      "entropy": 2.808818359375,
+      "epoch": 0.4166666666666667,
+      "grad_norm": 1.766530990600586,
+      "learning_rate": 6.56663069035493e-05,
+      "loss": 2.8823,
+      "mean_token_accuracy": 0.41536651641130445,
+      "num_tokens": 5372588.0,
+      "step": 6250
+    },
+    {
+      "entropy": 2.85748046875,
+      "epoch": 0.42,
+      "grad_norm": 2.2330243587493896,
+      "learning_rate": 6.515279230040856e-05,
+      "loss": 2.924,
+      "mean_token_accuracy": 0.4154829254746437,
+      "num_tokens": 5525866.0,
+      "step": 6300
+    },
+    {
+      "entropy": 2.813984375,
+      "epoch": 0.42333333333333334,
+      "grad_norm": 2.162248373031616,
+      "learning_rate": 6.46375116504208e-05,
+      "loss": 2.8969,
+      "mean_token_accuracy": 0.41659231394529345,
+      "num_tokens": 5677992.0,
+      "step": 6350
+    },
+    {
+      "entropy": 2.856396484375,
+      "epoch": 0.4266666666666667,
+      "grad_norm": 1.88718843460083,
+      "learning_rate": 6.412052500916845e-05,
+      "loss": 2.9383,
+      "mean_token_accuracy": 0.41758054718375204,
+      "num_tokens": 5831619.0,
+      "step": 6400
+    },
+    {
+      "entropy": 2.83671875,
+      "epoch": 0.43,
+      "grad_norm": 1.756526231765747,
+      "learning_rate": 6.360189263106599e-05,
+      "loss": 2.9082,
+      "mean_token_accuracy": 0.41605663910508156,
+      "num_tokens": 5984820.0,
+      "step": 6450
+    },
+    {
+      "entropy": 2.8180078125,
+      "epoch": 0.43333333333333335,
+      "grad_norm": 1.8252670764923096,
+      "learning_rate": 6.308167496233734e-05,
+      "loss": 2.8944,
+      "mean_token_accuracy": 0.4179633270204067,
+      "num_tokens": 6138386.0,
+      "step": 6500
+    },
+    {
+      "entropy": 2.811591796875,
+      "epoch": 0.43666666666666665,
+      "grad_norm": 4.292020320892334,
+      "learning_rate": 6.255993263397083e-05,
+      "loss": 2.8723,
+      "mean_token_accuracy": 0.4214246068894863,
+      "num_tokens": 6291806.0,
+      "step": 6550
+    },
+    {
+      "entropy": 2.8257421875,
+      "epoch": 0.44,
+      "grad_norm": 3.311739921569824,
+      "learning_rate": 6.20367264546528e-05,
+      "loss": 2.896,
+      "mean_token_accuracy": 0.41951198875904083,
+      "num_tokens": 6443998.0,
+      "step": 6600
+    },
+    {
+      "entropy": 2.8002734375,
+      "epoch": 0.44333333333333336,
+      "grad_norm": 1.8693840503692627,
+      "learning_rate": 6.151211740368034e-05,
+      "loss": 2.8698,
+      "mean_token_accuracy": 0.4244578213989735,
+      "num_tokens": 6597406.0,
+      "step": 6650
+    },
+    {
+      "entropy": 2.8307421875,
+      "epoch": 0.44666666666666666,
+      "grad_norm": 1.949150800704956,
+      "learning_rate": 6.098616662385419e-05,
+      "loss": 2.9105,
+      "mean_token_accuracy": 0.4161803536117077,
+      "num_tokens": 6750694.0,
+      "step": 6700
+    },
+    {
+      "entropy": 2.8480859375,
+      "epoch": 0.45,
+      "grad_norm": 1.7648441791534424,
+      "learning_rate": 6.0458935414352635e-05,
+      "loss": 2.925,
+      "mean_token_accuracy": 0.41687978282570837,
+      "num_tokens": 6903563.0,
+      "step": 6750
+    },
+    {
+      "entropy": 2.8136328125,
+      "epoch": 0.4533333333333333,
+      "grad_norm": 2.3119606971740723,
+      "learning_rate": 5.993048522358704e-05,
+      "loss": 2.881,
+      "mean_token_accuracy": 0.418667249083519,
+      "num_tokens": 7057216.0,
+      "step": 6800
+    },
+    {
+      "entropy": 2.8003369140625,
+      "epoch": 0.45666666666666667,
+      "grad_norm": 1.8502593040466309,
+      "learning_rate": 5.94008776420402e-05,
+      "loss": 2.8811,
+      "mean_token_accuracy": 0.4245522205531597,
+      "num_tokens": 7211517.0,
+      "step": 6850
+    },
+    {
+      "entropy": 2.8467724609375,
+      "epoch": 0.46,
+      "grad_norm": 3.208087682723999,
+      "learning_rate": 5.887017439508796e-05,
+      "loss": 2.9237,
+      "mean_token_accuracy": 0.4179289835691452,
+      "num_tokens": 7364204.0,
+      "step": 6900
+    },
+    {
+      "entropy": 2.7983544921875,
+      "epoch": 0.4633333333333333,
+      "grad_norm": 1.8808454275131226,
+      "learning_rate": 5.833843733580512e-05,
+      "loss": 2.8733,
+      "mean_token_accuracy": 0.4250455841422081,
+      "num_tokens": 7519008.0,
+      "step": 6950
+    },
+    {
+      "entropy": 2.781240234375,
+      "epoch": 0.4666666666666667,
+      "grad_norm": 3.435323476791382,
+      "learning_rate": 5.78057284377566e-05,
+      "loss": 2.837,
+      "mean_token_accuracy": 0.4327175974845886,
+      "num_tokens": 7671413.0,
+      "step": 7000
+    },
+    {
+      "entropy": 2.8481982421875,
+      "epoch": 0.47,
+      "grad_norm": 3.0656440258026123,
+      "learning_rate": 5.7272109787774354e-05,
+      "loss": 2.925,
+      "mean_token_accuracy": 0.4172740423679352,
+      "num_tokens": 7824483.0,
+      "step": 7050
+    },
+    {
+      "entropy": 2.809375,
+      "epoch": 0.47333333333333333,
+      "grad_norm": 2.2237229347229004,
+      "learning_rate": 5.6737643578721344e-05,
+      "loss": 2.9068,
+      "mean_token_accuracy": 0.42137662798166275,
+      "num_tokens": 7978009.0,
+      "step": 7100
+    },
+    {
+      "entropy": 2.84494140625,
+      "epoch": 0.4766666666666667,
+      "grad_norm": 1.875733733177185,
+      "learning_rate": 5.620239210224284e-05,
+      "loss": 2.923,
+      "mean_token_accuracy": 0.41358689829707146,
+      "num_tokens": 8130507.0,
+      "step": 7150
+    },
+    {
+      "entropy": 2.826162109375,
+      "epoch": 0.48,
+      "grad_norm": 1.910385251045227,
+      "learning_rate": 5.566641774150652e-05,
+      "loss": 2.8868,
+      "mean_token_accuracy": 0.41940944388508794,
+      "num_tokens": 8285593.0,
+      "step": 7200
+    },
+    {
+      "entropy": 2.7760546875,
+      "epoch": 0.48333333333333334,
+      "grad_norm": 2.746701955795288,
+      "learning_rate": 5.512978296393162e-05,
+      "loss": 2.8537,
+      "mean_token_accuracy": 0.42209434166550636,
+      "num_tokens": 8440418.0,
+      "step": 7250
+    },
+    {
+      "entropy": 2.8288330078125,
+      "epoch": 0.4866666666666667,
+      "grad_norm": 2.236929178237915,
+      "learning_rate": 5.459255031390855e-05,
+      "loss": 2.908,
+      "mean_token_accuracy": 0.41973535150289537,
+      "num_tokens": 8593754.0,
+      "step": 7300
+    },
+    {
+      "entropy": 2.8191015625,
+      "epoch": 0.49,
+      "grad_norm": 1.9573389291763306,
+      "learning_rate": 5.4054782405509285e-05,
+      "loss": 2.8843,
+      "mean_token_accuracy": 0.4191365033388138,
+      "num_tokens": 8746877.0,
+      "step": 7350
+    },
+    {
+      "entropy": 2.8084716796875,
+      "epoch": 0.49333333333333335,
+      "grad_norm": 2.003927230834961,
+      "learning_rate": 5.351654191518975e-05,
+      "loss": 2.8658,
+      "mean_token_accuracy": 0.4223553538322449,
+      "num_tokens": 8900625.0,
+      "step": 7400
+    },
+    {
+      "entropy": 2.816279296875,
+      "epoch": 0.49666666666666665,
+      "grad_norm": 2.671165943145752,
+      "learning_rate": 5.2977891574484994e-05,
+      "loss": 2.8967,
+      "mean_token_accuracy": 0.42066063448786734,
+      "num_tokens": 9054117.0,
+      "step": 7450
+    },
+    {
+      "entropy": 2.8142578125,
+      "epoch": 0.5,
+      "grad_norm": 2.034046173095703,
+      "learning_rate": 5.243889416269775e-05,
+      "loss": 2.8817,
+      "mean_token_accuracy": 0.4220552511513233,
+      "num_tokens": 9208425.0,
+      "step": 7500
+    },
+    {
+      "entropy": 2.809599609375,
+      "epoch": 0.5033333333333333,
+      "grad_norm": 2.3132572174072266,
+      "learning_rate": 5.189961249958174e-05,
+      "loss": 2.8701,
+      "mean_token_accuracy": 0.4232664656639099,
+      "num_tokens": 9360752.0,
+      "step": 7550
+    },
+    {
+      "entropy": 2.840048828125,
+      "epoch": 0.5066666666666667,
+      "grad_norm": 1.736668348312378,
+      "learning_rate": 5.136010943801989e-05,
+      "loss": 2.9123,
+      "mean_token_accuracy": 0.41717381954193117,
+      "num_tokens": 9514216.0,
+      "step": 7600
+    },
+    {
+      "entropy": 2.48101318359375,
+      "epoch": 0.51,
+      "grad_norm": 3.323343276977539,
+      "learning_rate": 5.082044785669897e-05,
+      "loss": 2.5754,
+      "mean_token_accuracy": 0.4657186330854893,
+      "num_tokens": 9663481.0,
+      "step": 7650
+    },
+    {
+      "entropy": 2.82603515625,
+      "epoch": 0.5133333333333333,
+      "grad_norm": 1.8825453519821167,
+      "learning_rate": 5.028069065278113e-05,
+      "loss": 2.9191,
+      "mean_token_accuracy": 0.41583023115992546,
+      "num_tokens": 9817354.0,
+      "step": 7700
+    },
+    {
+      "entropy": 2.7997119140625,
+      "epoch": 0.5166666666666667,
+      "grad_norm": 2.3555996417999268,
+      "learning_rate": 4.974090073457322e-05,
+      "loss": 2.8665,
+      "mean_token_accuracy": 0.42255289942026136,
+      "num_tokens": 9970062.0,
+      "step": 7750
+    },
+    {
+      "entropy": 2.8235595703125,
+      "epoch": 0.52,
+      "grad_norm": 1.8130338191986084,
+      "learning_rate": 4.9201141014194996e-05,
+      "loss": 2.896,
+      "mean_token_accuracy": 0.42268228754401205,
+      "num_tokens": 10123837.0,
+      "step": 7800
+    },
+    {
+      "entropy": 2.817529296875,
+      "epoch": 0.5233333333333333,
+      "grad_norm": 2.095828056335449,
+      "learning_rate": 4.8661474400246585e-05,
+      "loss": 2.9045,
+      "mean_token_accuracy": 0.41662930354475974,
+      "num_tokens": 10277512.0,
+      "step": 7850
+    },
+    {
+      "entropy": 2.8176708984375,
+      "epoch": 0.5266666666666666,
+      "grad_norm": 1.8423707485198975,
+      "learning_rate": 4.8121963790476716e-05,
+      "loss": 2.8869,
+      "mean_token_accuracy": 0.42100236147642134,
+      "num_tokens": 10431366.0,
+      "step": 7900
+    },
+    {
+      "entropy": 2.8667431640625,
+      "epoch": 0.53,
+      "grad_norm": 2.3884952068328857,
+      "learning_rate": 4.7582672064451866e-05,
+      "loss": 2.9367,
+      "mean_token_accuracy": 0.4157477006316185,
+      "num_tokens": 10585691.0,
+      "step": 7950
+    },
+    {
+      "entropy": 2.79511962890625,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.7394376993179321,
+      "learning_rate": 4.7043662076227837e-05,
+      "loss": 2.8729,
+      "mean_token_accuracy": 0.4245470950007439,
+      "num_tokens": 10739212.0,
+      "step": 8000
+    },
+    {
+      "entropy": 2.829560546875,
+      "epoch": 0.5366666666666666,
+      "grad_norm": 2.127032518386841,
+      "learning_rate": 4.6504996647024025e-05,
+      "loss": 2.8951,
+      "mean_token_accuracy": 0.42388937398791315,
+      "num_tokens": 153859.0,
+      "step": 8050
+    },
+    {
+      "entropy": 2.77587890625,
+      "epoch": 0.54,
+      "grad_norm": 2.321132183074951,
+      "learning_rate": 4.5966738557901814e-05,
+      "loss": 2.8512,
+      "mean_token_accuracy": 0.4274318239092827,
+      "num_tokens": 309366.0,
+      "step": 8100
+    },
+    {
+      "entropy": 2.764208984375,
+      "epoch": 0.5433333333333333,
+      "grad_norm": 2.4429869651794434,
+      "learning_rate": 4.5428950542447345e-05,
+      "loss": 2.8058,
+      "mean_token_accuracy": 0.4337243731319904,
+      "num_tokens": 463626.0,
+      "step": 8150
+    },
+    {
+      "entropy": 2.7914794921875,
+      "epoch": 0.5466666666666666,
+      "grad_norm": 4.270115852355957,
+      "learning_rate": 4.4891695279460004e-05,
+      "loss": 2.869,
+      "mean_token_accuracy": 0.42352987870574,
+      "num_tokens": 616986.0,
+      "step": 8200
+    },
+    {
+      "entropy": 2.797431640625,
+      "epoch": 0.55,
+      "grad_norm": 2.1567375659942627,
+      "learning_rate": 4.435503538564731e-05,
+      "loss": 2.8792,
+      "mean_token_accuracy": 0.4260627207159996,
+      "num_tokens": 771002.0,
+      "step": 8250
+    },
+    {
+      "entropy": 2.7621337890625,
+      "epoch": 0.5533333333333333,
+      "grad_norm": 2.77287220954895,
+      "learning_rate": 4.381903340832691e-05,
+      "loss": 2.8205,
+      "mean_token_accuracy": 0.4338701346516609,
+      "num_tokens": 924016.0,
+      "step": 8300
+    },
+    {
+      "entropy": 2.77218994140625,
+      "epoch": 0.5566666666666666,
+      "grad_norm": 1.9578077793121338,
+      "learning_rate": 4.3283751818136784e-05,
+      "loss": 2.8478,
+      "mean_token_accuracy": 0.43257290989160535,
+      "num_tokens": 1078531.0,
+      "step": 8350
+    },
+    {
+      "entropy": 2.808828125,
+      "epoch": 0.56,
+      "grad_norm": 2.0971014499664307,
+      "learning_rate": 4.2749253001754285e-05,
+      "loss": 2.8765,
+      "mean_token_accuracy": 0.42437156111001967,
+      "num_tokens": 1232268.0,
+      "step": 8400
+    },
+    {
+      "entropy": 2.7492041015625,
+      "epoch": 0.5633333333333334,
+      "grad_norm": 2.3967738151550293,
+      "learning_rate": 4.221559925462511e-05,
+      "loss": 2.8168,
+      "mean_token_accuracy": 0.42899366572499276,
+      "num_tokens": 1385942.0,
+      "step": 8450
+    },
+    {
+      "entropy": 2.83958984375,
+      "epoch": 0.5666666666666667,
+      "grad_norm": 2.1957943439483643,
+      "learning_rate": 4.1682852773702685e-05,
+      "loss": 2.9044,
+      "mean_token_accuracy": 0.4212977772951126,
+      "num_tokens": 1539407.0,
+      "step": 8500
+    },
+    {
+      "entropy": 0.7671748352050781,
+      "epoch": 0.57,
+      "grad_norm": 0.13969223201274872,
+      "learning_rate": 4.115107565019929e-05,
+      "loss": 0.7416,
+      "mean_token_accuracy": 0.8513110767304898,
+      "num_tokens": 1701093.0,
+      "step": 8550
+    },
+    {
+      "entropy": 2.7412037658691406,
+      "epoch": 0.5733333333333334,
+      "grad_norm": 5.486576557159424,
+      "learning_rate": 4.062032986234927e-05,
+      "loss": 2.8417,
+      "mean_token_accuracy": 0.43368106707930565,
+      "num_tokens": 1854202.0,
+      "step": 8600
+    },
+    {
+      "entropy": 2.7865234375,
+      "epoch": 0.5766666666666667,
+      "grad_norm": 2.4430174827575684,
+      "learning_rate": 4.009067726818556e-05,
+      "loss": 2.8568,
+      "mean_token_accuracy": 0.4283077374100685,
+      "num_tokens": 2007690.0,
+      "step": 8650
+    },
+    {
+      "entropy": 2.7960693359375,
+      "epoch": 0.58,
+      "grad_norm": 2.344949960708618,
+      "learning_rate": 3.956217959833017e-05,
+      "loss": 2.8723,
+      "mean_token_accuracy": 0.4265533179044724,
+      "num_tokens": 2161569.0,
+      "step": 8700
+    },
+    {
+      "entropy": 2.8105517578125,
+      "epoch": 0.5833333333333334,
+      "grad_norm": 3.076979398727417,
+      "learning_rate": 3.903489844879954e-05,
+      "loss": 2.8757,
+      "mean_token_accuracy": 0.4224506489932537,
+      "num_tokens": 2315122.0,
+      "step": 8750
+    },
+    {
+      "entropy": 2.742822265625,
+      "epoch": 0.5866666666666667,
+      "grad_norm": 2.2918591499328613,
+      "learning_rate": 3.850889527382554e-05,
+      "loss": 2.8181,
+      "mean_token_accuracy": 0.4310347574949265,
+      "num_tokens": 2469721.0,
+      "step": 8800
+    },
+    {
+      "entropy": 2.8107080078125,
+      "epoch": 0.59,
+      "grad_norm": 2.8631539344787598,
+      "learning_rate": 3.798423137869305e-05,
+      "loss": 2.8869,
+      "mean_token_accuracy": 0.42250056251883505,
+      "num_tokens": 2623034.0,
+      "step": 8850
+    },
+    {
+      "entropy": 2.7711279296875,
+      "epoch": 0.5933333333333334,
+      "grad_norm": 1.8896321058273315,
+      "learning_rate": 3.7460967912594845e-05,
+      "loss": 2.838,
+      "mean_token_accuracy": 0.4327010078728199,
+      "num_tokens": 2777019.0,
+      "step": 8900
+    },
+    {
+      "entropy": 2.8324755859375,
+      "epoch": 0.5966666666666667,
+      "grad_norm": 1.771895408630371,
+      "learning_rate": 3.693916586150479e-05,
+      "loss": 2.8983,
+      "mean_token_accuracy": 0.4137404492497444,
+      "num_tokens": 2930691.0,
+      "step": 8950
+    },
+    {
+      "entropy": 2.810810546875,
+      "epoch": 0.6,
+      "grad_norm": 2.086306095123291,
+      "learning_rate": 3.641888604106978e-05,
+      "loss": 2.8762,
+      "mean_token_accuracy": 0.4264249439537525,
+      "num_tokens": 3084012.0,
+      "step": 9000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 15000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 250,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.78220668076032e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fd10e5ed551017049c6cc19a2e941d32458df143b0746120dca9e19d5a7d10d
+size 6225