Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-70B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-70B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-70B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17f8624c970ddf995120555e61796d4df2ae4b395a5e20ef726c0a08962a30a3
+size 3313653480

chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

trainer_state.json

@@ -0,0 +1,2094 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 206,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.3063052594661713,
+      "epoch": 0.009708737864077669,
+      "grad_norm": 1.3077822923660278,
+      "learning_rate": 0.0,
+      "loss": 2.1177937984466553,
+      "mean_token_accuracy": 0.5264865458011627,
+      "num_tokens": 96695.0,
+      "step": 1
+    },
+    {
+      "entropy": 1.3945444822311401,
+      "epoch": 0.019417475728155338,
+      "grad_norm": 1.4227101802825928,
+      "learning_rate": 3.125e-06,
+      "loss": 2.264981746673584,
+      "mean_token_accuracy": 0.49969692155718803,
+      "num_tokens": 193992.0,
+      "step": 2
+    },
+    {
+      "entropy": 1.3749237358570099,
+      "epoch": 0.02912621359223301,
+      "grad_norm": 1.1497575044631958,
+      "learning_rate": 6.25e-06,
+      "loss": 2.0460948944091797,
+      "mean_token_accuracy": 0.5287926904857159,
+      "num_tokens": 291993.0,
+      "step": 3
+    },
+    {
+      "entropy": 1.4135093837976456,
+      "epoch": 0.038834951456310676,
+      "grad_norm": 1.0101280212402344,
+      "learning_rate": 9.375000000000001e-06,
+      "loss": 2.0115394592285156,
+      "mean_token_accuracy": 0.5312202051281929,
+      "num_tokens": 389906.0,
+      "step": 4
+    },
+    {
+      "entropy": 1.5345965921878815,
+      "epoch": 0.04854368932038835,
+      "grad_norm": 0.8249343037605286,
+      "learning_rate": 1.25e-05,
+      "loss": 1.9907541275024414,
+      "mean_token_accuracy": 0.5317840278148651,
+      "num_tokens": 487090.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.628997579216957,
+      "epoch": 0.05825242718446602,
+      "grad_norm": 0.6037535071372986,
+      "learning_rate": 1.5625e-05,
+      "loss": 1.917801856994629,
+      "mean_token_accuracy": 0.541858084499836,
+      "num_tokens": 584439.0,
+      "step": 6
+    },
+    {
+      "entropy": 1.8246440291404724,
+      "epoch": 0.06796116504854369,
+      "grad_norm": 0.3977610468864441,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 1.8268287181854248,
+      "mean_token_accuracy": 0.5501810759305954,
+      "num_tokens": 681933.0,
+      "step": 7
+    },
+    {
+      "entropy": 1.9396448135375977,
+      "epoch": 0.07766990291262135,
+      "grad_norm": 0.47858306765556335,
+      "learning_rate": 2.1875e-05,
+      "loss": 1.7852939367294312,
+      "mean_token_accuracy": 0.5552558451890945,
+      "num_tokens": 777167.0,
+      "step": 8
+    },
+    {
+      "entropy": 1.9606482535600662,
+      "epoch": 0.08737864077669903,
+      "grad_norm": 0.5253126621246338,
+      "learning_rate": 2.5e-05,
+      "loss": 1.7283756732940674,
+      "mean_token_accuracy": 0.5672201961278915,
+      "num_tokens": 875080.0,
+      "step": 9
+    },
+    {
+      "entropy": 1.9720240384340286,
+      "epoch": 0.0970873786407767,
+      "grad_norm": 0.5374228954315186,
+      "learning_rate": 2.8125000000000003e-05,
+      "loss": 1.711395263671875,
+      "mean_token_accuracy": 0.5681185722351074,
+      "num_tokens": 973099.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.82131689786911,
+      "epoch": 0.10679611650485436,
+      "grad_norm": 0.43059229850769043,
+      "learning_rate": 3.125e-05,
+      "loss": 1.6381919384002686,
+      "mean_token_accuracy": 0.5789104923605919,
+      "num_tokens": 1070082.0,
+      "step": 11
+    },
+    {
+      "entropy": 1.6750145703554153,
+      "epoch": 0.11650485436893204,
+      "grad_norm": 0.335764616727829,
+      "learning_rate": 3.4375e-05,
+      "loss": 1.5634382963180542,
+      "mean_token_accuracy": 0.5890125557780266,
+      "num_tokens": 1167345.0,
+      "step": 12
+    },
+    {
+      "entropy": 1.616977572441101,
+      "epoch": 0.1262135922330097,
+      "grad_norm": 0.29735782742500305,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 1.59102201461792,
+      "mean_token_accuracy": 0.5816583260893822,
+      "num_tokens": 1265097.0,
+      "step": 13
+    },
+    {
+      "entropy": 1.5444462895393372,
+      "epoch": 0.13592233009708737,
+      "grad_norm": 0.29059097170829773,
+      "learning_rate": 4.0625000000000005e-05,
+      "loss": 1.574419379234314,
+      "mean_token_accuracy": 0.5849687159061432,
+      "num_tokens": 1362414.0,
+      "step": 14
+    },
+    {
+      "entropy": 1.4560510218143463,
+      "epoch": 0.14563106796116504,
+      "grad_norm": 0.2826174199581146,
+      "learning_rate": 4.375e-05,
+      "loss": 1.5245388746261597,
+      "mean_token_accuracy": 0.5940257161855698,
+      "num_tokens": 1458384.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.4310729652643204,
+      "epoch": 0.1553398058252427,
+      "grad_norm": 0.2571217119693756,
+      "learning_rate": 4.6875e-05,
+      "loss": 1.493829607963562,
+      "mean_token_accuracy": 0.5997192710638046,
+      "num_tokens": 1555321.0,
+      "step": 16
+    },
+    {
+      "entropy": 1.3802899718284607,
+      "epoch": 0.1650485436893204,
+      "grad_norm": 0.24408026039600372,
+      "learning_rate": 5e-05,
+      "loss": 1.4393625259399414,
+      "mean_token_accuracy": 0.6103221923112869,
+      "num_tokens": 1652989.0,
+      "step": 17
+    },
+    {
+      "entropy": 1.423762321472168,
+      "epoch": 0.17475728155339806,
+      "grad_norm": 0.260315865278244,
+      "learning_rate": 4.999856295503635e-05,
+      "loss": 1.4357669353485107,
+      "mean_token_accuracy": 0.6085835695266724,
+      "num_tokens": 1749968.0,
+      "step": 18
+    },
+    {
+      "entropy": 1.4223755896091461,
+      "epoch": 0.18446601941747573,
+      "grad_norm": 0.26641297340393066,
+      "learning_rate": 4.999425198535325e-05,
+      "loss": 1.4089168310165405,
+      "mean_token_accuracy": 0.6123870462179184,
+      "num_tokens": 1847337.0,
+      "step": 19
+    },
+    {
+      "entropy": 1.4421514868736267,
+      "epoch": 0.1941747572815534,
+      "grad_norm": 0.2118070274591446,
+      "learning_rate": 4.998706758655528e-05,
+      "loss": 1.4083303213119507,
+      "mean_token_accuracy": 0.6158460900187492,
+      "num_tokens": 1944252.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.3758682906627655,
+      "epoch": 0.20388349514563106,
+      "grad_norm": 0.21211183071136475,
+      "learning_rate": 4.997701058458676e-05,
+      "loss": 1.3528016805648804,
+      "mean_token_accuracy": 0.6290531530976295,
+      "num_tokens": 2041362.0,
+      "step": 21
+    },
+    {
+      "entropy": 1.3912087827920914,
+      "epoch": 0.21359223300970873,
+      "grad_norm": 0.20889078080654144,
+      "learning_rate": 4.996408213563684e-05,
+      "loss": 1.3577451705932617,
+      "mean_token_accuracy": 0.6254735365509987,
+      "num_tokens": 2139370.0,
+      "step": 22
+    },
+    {
+      "entropy": 1.3215357810258865,
+      "epoch": 0.22330097087378642,
+      "grad_norm": 0.20261520147323608,
+      "learning_rate": 4.994828372600649e-05,
+      "loss": 1.300628662109375,
+      "mean_token_accuracy": 0.6371148228645325,
+      "num_tokens": 2233615.0,
+      "step": 23
+    },
+    {
+      "entropy": 1.3306704312562943,
+      "epoch": 0.23300970873786409,
+      "grad_norm": 0.19877074658870697,
+      "learning_rate": 4.9929617171937724e-05,
+      "loss": 1.3034857511520386,
+      "mean_token_accuracy": 0.6345249861478806,
+      "num_tokens": 2330947.0,
+      "step": 24
+    },
+    {
+      "entropy": 1.3362696319818497,
+      "epoch": 0.24271844660194175,
+      "grad_norm": 0.17692981660366058,
+      "learning_rate": 4.9908084619404735e-05,
+      "loss": 1.3098223209381104,
+      "mean_token_accuracy": 0.6328666806221008,
+      "num_tokens": 2428936.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.3557860553264618,
+      "epoch": 0.2524271844660194,
+      "grad_norm": 0.19237112998962402,
+      "learning_rate": 4.988368854386722e-05,
+      "loss": 1.3239411115646362,
+      "mean_token_accuracy": 0.6320360824465752,
+      "num_tokens": 2526310.0,
+      "step": 26
+    },
+    {
+      "entropy": 1.32492034137249,
+      "epoch": 0.2621359223300971,
+      "grad_norm": 0.19819748401641846,
+      "learning_rate": 4.985643174998578e-05,
+      "loss": 1.3037995100021362,
+      "mean_token_accuracy": 0.6353878080844879,
+      "num_tokens": 2623215.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.2510884702205658,
+      "epoch": 0.27184466019417475,
+      "grad_norm": 0.17776808142662048,
+      "learning_rate": 4.982631737129948e-05,
+      "loss": 1.2440943717956543,
+      "mean_token_accuracy": 0.6481084004044533,
+      "num_tokens": 2720683.0,
+      "step": 28
+    },
+    {
+      "entropy": 1.287326768040657,
+      "epoch": 0.2815533980582524,
+      "grad_norm": 0.17891287803649902,
+      "learning_rate": 4.9793348869865616e-05,
+      "loss": 1.29865300655365,
+      "mean_token_accuracy": 0.6353446468710899,
+      "num_tokens": 2818153.0,
+      "step": 29
+    },
+    {
+      "entropy": 1.2316195368766785,
+      "epoch": 0.2912621359223301,
+      "grad_norm": 0.17647312581539154,
+      "learning_rate": 4.9757530035861716e-05,
+      "loss": 1.253545880317688,
+      "mean_token_accuracy": 0.6457989513874054,
+      "num_tokens": 2915748.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.169528603553772,
+      "epoch": 0.30097087378640774,
+      "grad_norm": 0.16643674671649933,
+      "learning_rate": 4.971886498714977e-05,
+      "loss": 1.1838676929473877,
+      "mean_token_accuracy": 0.6634209230542183,
+      "num_tokens": 3013095.0,
+      "step": 31
+    },
+    {
+      "entropy": 1.2801441848278046,
+      "epoch": 0.3106796116504854,
+      "grad_norm": 0.17172685265541077,
+      "learning_rate": 4.967735816880286e-05,
+      "loss": 1.2642897367477417,
+      "mean_token_accuracy": 0.6400710195302963,
+      "num_tokens": 3110631.0,
+      "step": 32
+    },
+    {
+      "entropy": 1.2720160782337189,
+      "epoch": 0.32038834951456313,
+      "grad_norm": 0.16451463103294373,
+      "learning_rate": 4.963301435259413e-05,
+      "loss": 1.2586028575897217,
+      "mean_token_accuracy": 0.6447242349386215,
+      "num_tokens": 3207970.0,
+      "step": 33
+    },
+    {
+      "entropy": 1.2911356091499329,
+      "epoch": 0.3300970873786408,
+      "grad_norm": 0.1695583015680313,
+      "learning_rate": 4.95858386364482e-05,
+      "loss": 1.2504911422729492,
+      "mean_token_accuracy": 0.6436235383152962,
+      "num_tokens": 3305342.0,
+      "step": 34
+    },
+    {
+      "entropy": 1.2505649775266647,
+      "epoch": 0.33980582524271846,
+      "grad_norm": 0.1600319743156433,
+      "learning_rate": 4.9535836443855096e-05,
+      "loss": 1.2131391763687134,
+      "mean_token_accuracy": 0.6537701711058617,
+      "num_tokens": 3399739.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.1979082226753235,
+      "epoch": 0.34951456310679613,
+      "grad_norm": 0.15272502601146698,
+      "learning_rate": 4.948301352324673e-05,
+      "loss": 1.1704742908477783,
+      "mean_token_accuracy": 0.6647882163524628,
+      "num_tokens": 3497430.0,
+      "step": 36
+    },
+    {
+      "entropy": 1.2117689549922943,
+      "epoch": 0.3592233009708738,
+      "grad_norm": 0.15061117708683014,
+      "learning_rate": 4.942737594733609e-05,
+      "loss": 1.1953562498092651,
+      "mean_token_accuracy": 0.6564236953854561,
+      "num_tokens": 3593935.0,
+      "step": 37
+    },
+    {
+      "entropy": 1.1917639821767807,
+      "epoch": 0.36893203883495146,
+      "grad_norm": 0.15709325671195984,
+      "learning_rate": 4.9368930112419e-05,
+      "loss": 1.1850690841674805,
+      "mean_token_accuracy": 0.660385861992836,
+      "num_tokens": 3691118.0,
+      "step": 38
+    },
+    {
+      "entropy": 1.2446843981742859,
+      "epoch": 0.3786407766990291,
+      "grad_norm": 0.16046260297298431,
+      "learning_rate": 4.930768273763889e-05,
+      "loss": 1.2477033138275146,
+      "mean_token_accuracy": 0.6435192674398422,
+      "num_tokens": 3788648.0,
+      "step": 39
+    },
+    {
+      "entropy": 1.2205789387226105,
+      "epoch": 0.3883495145631068,
+      "grad_norm": 0.1660885363817215,
+      "learning_rate": 4.92436408642143e-05,
+      "loss": 1.2170484066009521,
+      "mean_token_accuracy": 0.6503675952553749,
+      "num_tokens": 3886532.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.205104410648346,
+      "epoch": 0.39805825242718446,
+      "grad_norm": 0.15774689614772797,
+      "learning_rate": 4.917681185462934e-05,
+      "loss": 1.207252860069275,
+      "mean_token_accuracy": 0.655083142220974,
+      "num_tokens": 3984443.0,
+      "step": 41
+    },
+    {
+      "entropy": 1.1894231885671616,
+      "epoch": 0.4077669902912621,
+      "grad_norm": 0.1520465910434723,
+      "learning_rate": 4.910720339178735e-05,
+      "loss": 1.1727756261825562,
+      "mean_token_accuracy": 0.6604171320796013,
+      "num_tokens": 4082127.0,
+      "step": 42
+    },
+    {
+      "entropy": 1.1813564747571945,
+      "epoch": 0.4174757281553398,
+      "grad_norm": 0.1579592078924179,
+      "learning_rate": 4.90348234781276e-05,
+      "loss": 1.1708219051361084,
+      "mean_token_accuracy": 0.6622453778982162,
+      "num_tokens": 4179696.0,
+      "step": 43
+    },
+    {
+      "entropy": 1.1847212761640549,
+      "epoch": 0.42718446601941745,
+      "grad_norm": 0.16752171516418457,
+      "learning_rate": 4.895968043470532e-05,
+      "loss": 1.1717603206634521,
+      "mean_token_accuracy": 0.6604401022195816,
+      "num_tokens": 4276989.0,
+      "step": 44
+    },
+    {
+      "entropy": 1.1232565939426422,
+      "epoch": 0.4368932038834951,
+      "grad_norm": 0.16498398780822754,
+      "learning_rate": 4.8881782900235094e-05,
+      "loss": 1.1282835006713867,
+      "mean_token_accuracy": 0.6696558594703674,
+      "num_tokens": 4374513.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.1546286791563034,
+      "epoch": 0.44660194174757284,
+      "grad_norm": 0.16002920269966125,
+      "learning_rate": 4.880113983009768e-05,
+      "loss": 1.163672924041748,
+      "mean_token_accuracy": 0.6636269986629486,
+      "num_tokens": 4472543.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.220591351389885,
+      "epoch": 0.4563106796116505,
+      "grad_norm": 0.15611808001995087,
+      "learning_rate": 4.87177604953105e-05,
+      "loss": 1.227357268333435,
+      "mean_token_accuracy": 0.6496708914637566,
+      "num_tokens": 4570196.0,
+      "step": 47
+    },
+    {
+      "entropy": 1.1873502284288406,
+      "epoch": 0.46601941747572817,
+      "grad_norm": 0.15429329872131348,
+      "learning_rate": 4.86316544814618e-05,
+      "loss": 1.1806132793426514,
+      "mean_token_accuracy": 0.6577019914984703,
+      "num_tokens": 4667589.0,
+      "step": 48
+    },
+    {
+      "entropy": 1.1497704908251762,
+      "epoch": 0.47572815533980584,
+      "grad_norm": 0.14651840925216675,
+      "learning_rate": 4.854283168760868e-05,
+      "loss": 1.132278323173523,
+      "mean_token_accuracy": 0.6702492162585258,
+      "num_tokens": 4765775.0,
+      "step": 49
+    },
+    {
+      "entropy": 1.19112890958786,
+      "epoch": 0.4854368932038835,
+      "grad_norm": 0.16413071751594543,
+      "learning_rate": 4.8451302325139004e-05,
+      "loss": 1.1689636707305908,
+      "mean_token_accuracy": 0.6610261425375938,
+      "num_tokens": 4864001.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1795462369918823,
+      "epoch": 0.49514563106796117,
+      "grad_norm": 0.1465304046869278,
+      "learning_rate": 4.835707691659753e-05,
+      "loss": 1.1613165140151978,
+      "mean_token_accuracy": 0.6660530865192413,
+      "num_tokens": 4960993.0,
+      "step": 51
+    },
+    {
+      "entropy": 1.1446045935153961,
+      "epoch": 0.5048543689320388,
+      "grad_norm": 0.1478618085384369,
+      "learning_rate": 4.8260166294476164e-05,
+      "loss": 1.1292091608047485,
+      "mean_token_accuracy": 0.6699203625321388,
+      "num_tokens": 5058285.0,
+      "step": 52
+    },
+    {
+      "entropy": 1.1335399597883224,
+      "epoch": 0.5145631067961165,
+      "grad_norm": 0.15407773852348328,
+      "learning_rate": 4.8160581599968625e-05,
+      "loss": 1.1282105445861816,
+      "mean_token_accuracy": 0.6706565544009209,
+      "num_tokens": 5155340.0,
+      "step": 53
+    },
+    {
+      "entropy": 1.1364393010735512,
+      "epoch": 0.5242718446601942,
+      "grad_norm": 0.15273907780647278,
+      "learning_rate": 4.8058334281689595e-05,
+      "loss": 1.1344889402389526,
+      "mean_token_accuracy": 0.668300211429596,
+      "num_tokens": 5253290.0,
+      "step": 54
+    },
+    {
+      "entropy": 1.1561526656150818,
+      "epoch": 0.5339805825242718,
+      "grad_norm": 0.15070948004722595,
+      "learning_rate": 4.7953436094358595e-05,
+      "loss": 1.145023226737976,
+      "mean_token_accuracy": 0.6662660464644432,
+      "num_tokens": 5350631.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.1212447136640549,
+      "epoch": 0.5436893203883495,
+      "grad_norm": 0.1489776223897934,
+      "learning_rate": 4.784589909744855e-05,
+      "loss": 1.1079856157302856,
+      "mean_token_accuracy": 0.6758884415030479,
+      "num_tokens": 5447338.0,
+      "step": 56
+    },
+    {
+      "entropy": 1.1728671193122864,
+      "epoch": 0.5533980582524272,
+      "grad_norm": 0.1548493206501007,
+      "learning_rate": 4.7735735653799463e-05,
+      "loss": 1.159340739250183,
+      "mean_token_accuracy": 0.6629773899912834,
+      "num_tokens": 5543513.0,
+      "step": 57
+    },
+    {
+      "entropy": 1.1602209657430649,
+      "epoch": 0.5631067961165048,
+      "grad_norm": 0.15153397619724274,
+      "learning_rate": 4.762295842819707e-05,
+      "loss": 1.1558430194854736,
+      "mean_token_accuracy": 0.6631080582737923,
+      "num_tokens": 5640853.0,
+      "step": 58
+    },
+    {
+      "entropy": 1.167491763830185,
+      "epoch": 0.5728155339805825,
+      "grad_norm": 0.15950381755828857,
+      "learning_rate": 4.75075803859169e-05,
+      "loss": 1.1687371730804443,
+      "mean_token_accuracy": 0.662310965359211,
+      "num_tokens": 5737838.0,
+      "step": 59
+    },
+    {
+      "entropy": 1.1121539771556854,
+      "epoch": 0.5825242718446602,
+      "grad_norm": 0.15362070500850677,
+      "learning_rate": 4.7389614791233726e-05,
+      "loss": 1.122381567955017,
+      "mean_token_accuracy": 0.6725770682096481,
+      "num_tokens": 5835215.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1065661907196045,
+      "epoch": 0.5922330097087378,
+      "grad_norm": 0.15264824032783508,
+      "learning_rate": 4.726907520589664e-05,
+      "loss": 1.1158289909362793,
+      "mean_token_accuracy": 0.673554889857769,
+      "num_tokens": 5933071.0,
+      "step": 61
+    },
+    {
+      "entropy": 1.1254866421222687,
+      "epoch": 0.6019417475728155,
+      "grad_norm": 0.1510513871908188,
+      "learning_rate": 4.714597548756996e-05,
+      "loss": 1.1177195310592651,
+      "mean_token_accuracy": 0.6726218238472939,
+      "num_tokens": 6030510.0,
+      "step": 62
+    },
+    {
+      "entropy": 1.1632820963859558,
+      "epoch": 0.6116504854368932,
+      "grad_norm": 0.15587088465690613,
+      "learning_rate": 4.702032978824011e-05,
+      "loss": 1.1515460014343262,
+      "mean_token_accuracy": 0.6640657410025597,
+      "num_tokens": 6128308.0,
+      "step": 63
+    },
+    {
+      "entropy": 1.13534314930439,
+      "epoch": 0.6213592233009708,
+      "grad_norm": 0.15413475036621094,
+      "learning_rate": 4.6892152552588655e-05,
+      "loss": 1.1225864887237549,
+      "mean_token_accuracy": 0.6697054132819176,
+      "num_tokens": 6225307.0,
+      "step": 64
+    },
+    {
+      "entropy": 1.1529878377914429,
+      "epoch": 0.6310679611650486,
+      "grad_norm": 0.14912283420562744,
+      "learning_rate": 4.6761458516331655e-05,
+      "loss": 1.137518048286438,
+      "mean_token_accuracy": 0.6690729334950447,
+      "num_tokens": 6322345.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.162615068256855,
+      "epoch": 0.6407766990291263,
+      "grad_norm": 0.15968185663223267,
+      "learning_rate": 4.662826270452565e-05,
+      "loss": 1.148972988128662,
+      "mean_token_accuracy": 0.6663458272814751,
+      "num_tokens": 6419830.0,
+      "step": 66
+    },
+    {
+      "entropy": 1.1012173295021057,
+      "epoch": 0.6504854368932039,
+      "grad_norm": 0.15278087556362152,
+      "learning_rate": 4.649258042984026e-05,
+      "loss": 1.0935297012329102,
+      "mean_token_accuracy": 0.6765975803136826,
+      "num_tokens": 6517180.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.1730160862207413,
+      "epoch": 0.6601941747572816,
+      "grad_norm": 0.15458795428276062,
+      "learning_rate": 4.6354427290797875e-05,
+      "loss": 1.1597059965133667,
+      "mean_token_accuracy": 0.6620247736573219,
+      "num_tokens": 6614441.0,
+      "step": 68
+    },
+    {
+      "entropy": 1.1525196582078934,
+      "epoch": 0.6699029126213593,
+      "grad_norm": 0.15900148451328278,
+      "learning_rate": 4.621381916998029e-05,
+      "loss": 1.1408532857894897,
+      "mean_token_accuracy": 0.6672633439302444,
+      "num_tokens": 6711921.0,
+      "step": 69
+    },
+    {
+      "entropy": 1.1234073489904404,
+      "epoch": 0.6796116504854369,
+      "grad_norm": 0.9614858031272888,
+      "learning_rate": 4.607077223220285e-05,
+      "loss": 1.1134552955627441,
+      "mean_token_accuracy": 0.6734683215618134,
+      "num_tokens": 6808916.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.1003537625074387,
+      "epoch": 0.6893203883495146,
+      "grad_norm": 0.27281081676483154,
+      "learning_rate": 4.592530292265609e-05,
+      "loss": 1.093156337738037,
+      "mean_token_accuracy": 0.677058070898056,
+      "num_tokens": 6905733.0,
+      "step": 71
+    },
+    {
+      "entropy": 1.1193806380033493,
+      "epoch": 0.6990291262135923,
+      "grad_norm": 0.15709951519966125,
+      "learning_rate": 4.5777427965015096e-05,
+      "loss": 1.1156103610992432,
+      "mean_token_accuracy": 0.6733296439051628,
+      "num_tokens": 7002459.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.0762626677751541,
+      "epoch": 0.7087378640776699,
+      "grad_norm": 0.14975018799304962,
+      "learning_rate": 4.562716435951692e-05,
+      "loss": 1.0788187980651855,
+      "mean_token_accuracy": 0.6805250495672226,
+      "num_tokens": 7100490.0,
+      "step": 73
+    },
+    {
+      "entropy": 1.1070052459836006,
+      "epoch": 0.7184466019417476,
+      "grad_norm": 0.163674458861351,
+      "learning_rate": 4.5474529381006146e-05,
+      "loss": 1.1094239950180054,
+      "mean_token_accuracy": 0.6733985841274261,
+      "num_tokens": 7197967.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.133701667189598,
+      "epoch": 0.7281553398058253,
+      "grad_norm": 0.16219182312488556,
+      "learning_rate": 4.531954057694897e-05,
+      "loss": 1.131150484085083,
+      "mean_token_accuracy": 0.6676085367798805,
+      "num_tokens": 7292478.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.151291936635971,
+      "epoch": 0.7378640776699029,
+      "grad_norm": 0.15997757017612457,
+      "learning_rate": 4.516221576541581e-05,
+      "loss": 1.1296626329421997,
+      "mean_token_accuracy": 0.6666271537542343,
+      "num_tokens": 7389056.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.1276646926999092,
+      "epoch": 0.7475728155339806,
+      "grad_norm": 0.16011382639408112,
+      "learning_rate": 4.5002573033032904e-05,
+      "loss": 1.1120346784591675,
+      "mean_token_accuracy": 0.6736837849020958,
+      "num_tokens": 7486257.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.115042731165886,
+      "epoch": 0.7572815533980582,
+      "grad_norm": 0.1535303145647049,
+      "learning_rate": 4.484063073290301e-05,
+      "loss": 1.1091301441192627,
+      "mean_token_accuracy": 0.6762245669960976,
+      "num_tokens": 7583265.0,
+      "step": 78
+    },
+    {
+      "entropy": 1.087426796555519,
+      "epoch": 0.7669902912621359,
+      "grad_norm": 0.16656380891799927,
+      "learning_rate": 4.467640748249548e-05,
+      "loss": 1.0862419605255127,
+      "mean_token_accuracy": 0.6805471926927567,
+      "num_tokens": 7680175.0,
+      "step": 79
+    },
+    {
+      "entropy": 1.1316653937101364,
+      "epoch": 0.7766990291262136,
+      "grad_norm": 0.15928491950035095,
+      "learning_rate": 4.4509922161505926e-05,
+      "loss": 1.1360259056091309,
+      "mean_token_accuracy": 0.6669512763619423,
+      "num_tokens": 7777798.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.1151190549135208,
+      "epoch": 0.7864077669902912,
+      "grad_norm": 0.15734143555164337,
+      "learning_rate": 4.4341193909685686e-05,
+      "loss": 1.1140034198760986,
+      "mean_token_accuracy": 0.6693683862686157,
+      "num_tokens": 7875637.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.1036027371883392,
+      "epoch": 0.7961165048543689,
+      "grad_norm": 0.16518977284431458,
+      "learning_rate": 4.417024212464152e-05,
+      "loss": 1.0992156267166138,
+      "mean_token_accuracy": 0.6753587499260902,
+      "num_tokens": 7973214.0,
+      "step": 82
+    },
+    {
+      "entropy": 1.1240820437669754,
+      "epoch": 0.8058252427184466,
+      "grad_norm": 0.15742117166519165,
+      "learning_rate": 4.399708645960559e-05,
+      "loss": 1.1219127178192139,
+      "mean_token_accuracy": 0.6707674041390419,
+      "num_tokens": 8071015.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.0782043635845184,
+      "epoch": 0.8155339805825242,
+      "grad_norm": 0.15135641396045685,
+      "learning_rate": 4.382174682117598e-05,
+      "loss": 1.0685319900512695,
+      "mean_token_accuracy": 0.6833792477846146,
+      "num_tokens": 8167887.0,
+      "step": 84
+    },
+    {
+      "entropy": 1.1078387796878815,
+      "epoch": 0.8252427184466019,
+      "grad_norm": 0.15460753440856934,
+      "learning_rate": 4.364424336702825e-05,
+      "loss": 1.1065967082977295,
+      "mean_token_accuracy": 0.6753173843026161,
+      "num_tokens": 8265328.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.109467014670372,
+      "epoch": 0.8349514563106796,
+      "grad_norm": 0.15673620998859406,
+      "learning_rate": 4.346459650359798e-05,
+      "loss": 1.1054904460906982,
+      "mean_token_accuracy": 0.6707606092095375,
+      "num_tokens": 8362790.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.125985011458397,
+      "epoch": 0.8446601941747572,
+      "grad_norm": 0.1561659872531891,
+      "learning_rate": 4.328282688373479e-05,
+      "loss": 1.111283302307129,
+      "mean_token_accuracy": 0.6752806529402733,
+      "num_tokens": 8460119.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.0695659816265106,
+      "epoch": 0.8543689320388349,
+      "grad_norm": 0.16007103025913239,
+      "learning_rate": 4.3098955404328045e-05,
+      "loss": 1.056589961051941,
+      "mean_token_accuracy": 0.6865223124623299,
+      "num_tokens": 8555768.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.0883630514144897,
+      "epoch": 0.8640776699029126,
+      "grad_norm": 0.15864451229572296,
+      "learning_rate": 4.29130032039044e-05,
+      "loss": 1.0721038579940796,
+      "mean_token_accuracy": 0.6814716532826424,
+      "num_tokens": 8653989.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.0660209506750107,
+      "epoch": 0.8737864077669902,
+      "grad_norm": 0.15096718072891235,
+      "learning_rate": 4.272499166019771e-05,
+      "loss": 1.0504462718963623,
+      "mean_token_accuracy": 0.6850994229316711,
+      "num_tokens": 8751995.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1066109761595726,
+      "epoch": 0.883495145631068,
+      "grad_norm": 0.16408860683441162,
+      "learning_rate": 4.253494238769134e-05,
+      "loss": 1.106789231300354,
+      "mean_token_accuracy": 0.6758328899741173,
+      "num_tokens": 8848370.0,
+      "step": 91
+    },
+    {
+      "entropy": 1.0865231901407242,
+      "epoch": 0.8932038834951457,
+      "grad_norm": 0.1649610996246338,
+      "learning_rate": 4.234287723513326e-05,
+      "loss": 1.095716118812561,
+      "mean_token_accuracy": 0.6755226477980614,
+      "num_tokens": 8946065.0,
+      "step": 92
+    },
+    {
+      "entropy": 1.0563477203249931,
+      "epoch": 0.9029126213592233,
+      "grad_norm": 0.15605774521827698,
+      "learning_rate": 4.2148818283024304e-05,
+      "loss": 1.0641911029815674,
+      "mean_token_accuracy": 0.684117816388607,
+      "num_tokens": 9043734.0,
+      "step": 93
+    },
+    {
+      "entropy": 1.1027402877807617,
+      "epoch": 0.912621359223301,
+      "grad_norm": 0.1745792031288147,
+      "learning_rate": 4.195278784107964e-05,
+      "loss": 1.093820571899414,
+      "mean_token_accuracy": 0.6766936853528023,
+      "num_tokens": 9141101.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.1108492761850357,
+      "epoch": 0.9223300970873787,
+      "grad_norm": 0.1714155375957489,
+      "learning_rate": 4.175480844566404e-05,
+      "loss": 1.0948399305343628,
+      "mean_token_accuracy": 0.6759923100471497,
+      "num_tokens": 9238764.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.092107117176056,
+      "epoch": 0.9320388349514563,
+      "grad_norm": 0.16130048036575317,
+      "learning_rate": 4.1554902857200924e-05,
+      "loss": 1.0810198783874512,
+      "mean_token_accuracy": 0.6786670908331871,
+      "num_tokens": 9335019.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.0851488932967186,
+      "epoch": 0.941747572815534,
+      "grad_norm": 0.16832125186920166,
+      "learning_rate": 4.135309405755583e-05,
+      "loss": 1.073775291442871,
+      "mean_token_accuracy": 0.6807554513216019,
+      "num_tokens": 9433084.0,
+      "step": 97
+    },
+    {
+      "entropy": 1.0768142491579056,
+      "epoch": 0.9514563106796117,
+      "grad_norm": 0.16131848096847534,
+      "learning_rate": 4.11494052473943e-05,
+      "loss": 1.0637844800949097,
+      "mean_token_accuracy": 0.6831372752785683,
+      "num_tokens": 9530495.0,
+      "step": 98
+    },
+    {
+      "entropy": 1.063091091811657,
+      "epoch": 0.9611650485436893,
+      "grad_norm": 0.16142332553863525,
+      "learning_rate": 4.094385984351462e-05,
+      "loss": 1.0600042343139648,
+      "mean_token_accuracy": 0.6849533692002296,
+      "num_tokens": 9628209.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.0749415531754494,
+      "epoch": 0.970873786407767,
+      "grad_norm": 0.18972496688365936,
+      "learning_rate": 4.073648147615579e-05,
+      "loss": 1.0742400884628296,
+      "mean_token_accuracy": 0.6798612177371979,
+      "num_tokens": 9726165.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.0456555485725403,
+      "epoch": 0.9805825242718447,
+      "grad_norm": 0.1582217514514923,
+      "learning_rate": 4.052729398628089e-05,
+      "loss": 1.0480663776397705,
+      "mean_token_accuracy": 0.6852379813790321,
+      "num_tokens": 9823616.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.0736559927463531,
+      "epoch": 0.9902912621359223,
+      "grad_norm": 0.16985982656478882,
+      "learning_rate": 4.031632142283622e-05,
+      "loss": 1.0799673795700073,
+      "mean_token_accuracy": 0.6796677485108376,
+      "num_tokens": 9921527.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.0745692774653435,
+      "epoch": 1.0,
+      "grad_norm": 0.16134285926818848,
+      "learning_rate": 4.0103588039986556e-05,
+      "loss": 1.078010082244873,
+      "mean_token_accuracy": 0.6782025918364525,
+      "num_tokens": 10018894.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.052033707499504,
+      "epoch": 1.0097087378640777,
+      "grad_norm": 0.15611664950847626,
+      "learning_rate": 3.988911829432682e-05,
+      "loss": 1.0273964405059814,
+      "mean_token_accuracy": 0.6917483061552048,
+      "num_tokens": 10116408.0,
+      "step": 104
+    },
+    {
+      "entropy": 1.106005534529686,
+      "epoch": 1.0194174757281553,
+      "grad_norm": 0.18703246116638184,
+      "learning_rate": 3.967293684207042e-05,
+      "loss": 1.07395339012146,
+      "mean_token_accuracy": 0.6806365475058556,
+      "num_tokens": 10213505.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.0383973494172096,
+      "epoch": 1.029126213592233,
+      "grad_norm": 0.16569890081882477,
+      "learning_rate": 3.945506853621476e-05,
+      "loss": 1.0093125104904175,
+      "mean_token_accuracy": 0.6949852854013443,
+      "num_tokens": 10311543.0,
+      "step": 106
+    },
+    {
+      "entropy": 1.027927227318287,
+      "epoch": 1.0388349514563107,
+      "grad_norm": 0.15815988183021545,
+      "learning_rate": 3.923553842368396e-05,
+      "loss": 1.0087497234344482,
+      "mean_token_accuracy": 0.6973673179745674,
+      "num_tokens": 10408148.0,
+      "step": 107
+    },
+    {
+      "entropy": 1.0794779807329178,
+      "epoch": 1.0485436893203883,
+      "grad_norm": 0.16736039519309998,
+      "learning_rate": 3.901437174244943e-05,
+      "loss": 1.0720303058624268,
+      "mean_token_accuracy": 0.6798778995871544,
+      "num_tokens": 10505638.0,
+      "step": 108
+    },
+    {
+      "entropy": 1.0462391898036003,
+      "epoch": 1.058252427184466,
+      "grad_norm": 0.17532268166542053,
+      "learning_rate": 3.879159391862839e-05,
+      "loss": 1.0518163442611694,
+      "mean_token_accuracy": 0.6836148276925087,
+      "num_tokens": 10602743.0,
+      "step": 109
+    },
+    {
+      "entropy": 1.0277544036507607,
+      "epoch": 1.0679611650485437,
+      "grad_norm": 0.16957585513591766,
+      "learning_rate": 3.856723056356084e-05,
+      "loss": 1.0378530025482178,
+      "mean_token_accuracy": 0.6870032772421837,
+      "num_tokens": 10700300.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.0463587269186974,
+      "epoch": 1.0776699029126213,
+      "grad_norm": 0.16475018858909607,
+      "learning_rate": 3.834130747086512e-05,
+      "loss": 1.0512408018112183,
+      "mean_token_accuracy": 0.6838105544447899,
+      "num_tokens": 10797991.0,
+      "step": 111
+    },
+    {
+      "entropy": 1.0562317967414856,
+      "epoch": 1.087378640776699,
+      "grad_norm": 0.17692257463932037,
+      "learning_rate": 3.811385061347263e-05,
+      "loss": 1.045755386352539,
+      "mean_token_accuracy": 0.6854795292019844,
+      "num_tokens": 10895950.0,
+      "step": 112
+    },
+    {
+      "entropy": 1.0278217941522598,
+      "epoch": 1.0970873786407767,
+      "grad_norm": 0.1635172963142395,
+      "learning_rate": 3.788488614064188e-05,
+      "loss": 1.0115208625793457,
+      "mean_token_accuracy": 0.6944800242781639,
+      "num_tokens": 10992915.0,
+      "step": 113
+    },
+    {
+      "entropy": 1.070695973932743,
+      "epoch": 1.1067961165048543,
+      "grad_norm": 0.17553620040416718,
+      "learning_rate": 3.7654440374952286e-05,
+      "loss": 1.0541203022003174,
+      "mean_token_accuracy": 0.6825162619352341,
+      "num_tokens": 11090381.0,
+      "step": 114
+    },
+    {
+      "entropy": 1.007257767021656,
+      "epoch": 1.116504854368932,
+      "grad_norm": 0.15791647136211395,
+      "learning_rate": 3.742253980927799e-05,
+      "loss": 0.9898063540458679,
+      "mean_token_accuracy": 0.6973354369401932,
+      "num_tokens": 11188346.0,
+      "step": 115
+    },
+    {
+      "entropy": 1.0168157815933228,
+      "epoch": 1.1262135922330097,
+      "grad_norm": 0.17020469903945923,
+      "learning_rate": 3.7189211103742206e-05,
+      "loss": 1.0183147192001343,
+      "mean_token_accuracy": 0.6948697343468666,
+      "num_tokens": 11286538.0,
+      "step": 116
+    },
+    {
+      "entropy": 1.01011723279953,
+      "epoch": 1.1359223300970873,
+      "grad_norm": 0.1641688495874405,
+      "learning_rate": 3.695448108265221e-05,
+      "loss": 1.0146812200546265,
+      "mean_token_accuracy": 0.69698815792799,
+      "num_tokens": 11384057.0,
+      "step": 117
+    },
+    {
+      "entropy": 0.9920699745416641,
+      "epoch": 1.145631067961165,
+      "grad_norm": 0.16153588891029358,
+      "learning_rate": 3.671837673141559e-05,
+      "loss": 0.9923903346061707,
+      "mean_token_accuracy": 0.7004250958561897,
+      "num_tokens": 11481978.0,
+      "step": 118
+    },
+    {
+      "entropy": 1.055176705121994,
+      "epoch": 1.1553398058252426,
+      "grad_norm": 0.1622096300125122,
+      "learning_rate": 3.648092519343783e-05,
+      "loss": 1.0450016260147095,
+      "mean_token_accuracy": 0.6863952726125717,
+      "num_tokens": 11579536.0,
+      "step": 119
+    },
+    {
+      "entropy": 1.0694908052682877,
+      "epoch": 1.1650485436893203,
+      "grad_norm": 0.17806941270828247,
+      "learning_rate": 3.6242153767001895e-05,
+      "loss": 1.0527136325836182,
+      "mean_token_accuracy": 0.6836417689919472,
+      "num_tokens": 11673620.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.0538558959960938,
+      "epoch": 1.174757281553398,
+      "grad_norm": 0.21832379698753357,
+      "learning_rate": 3.600208990212984e-05,
+      "loss": 1.0327177047729492,
+      "mean_token_accuracy": 0.6883136481046677,
+      "num_tokens": 11771212.0,
+      "step": 121
+    },
+    {
+      "entropy": 1.0258601680397987,
+      "epoch": 1.1844660194174756,
+      "grad_norm": 0.17365030944347382,
+      "learning_rate": 3.5760761197427095e-05,
+      "loss": 1.026971697807312,
+      "mean_token_accuracy": 0.6886790469288826,
+      "num_tokens": 11868700.0,
+      "step": 122
+    },
+    {
+      "entropy": 1.0418048202991486,
+      "epoch": 1.1941747572815533,
+      "grad_norm": 0.16434766352176666,
+      "learning_rate": 3.551819539690965e-05,
+      "loss": 1.0223963260650635,
+      "mean_token_accuracy": 0.6908348724246025,
+      "num_tokens": 11965887.0,
+      "step": 123
+    },
+    {
+      "entropy": 1.0289766415953636,
+      "epoch": 1.203883495145631,
+      "grad_norm": 0.1707381159067154,
+      "learning_rate": 3.527442038681446e-05,
+      "loss": 1.020445466041565,
+      "mean_token_accuracy": 0.6910003051161766,
+      "num_tokens": 12063024.0,
+      "step": 124
+    },
+    {
+      "entropy": 1.0090995877981186,
+      "epoch": 1.2135922330097086,
+      "grad_norm": 0.16638465225696564,
+      "learning_rate": 3.5029464192393555e-05,
+      "loss": 1.0045936107635498,
+      "mean_token_accuracy": 0.697821132838726,
+      "num_tokens": 12161173.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.0521429032087326,
+      "epoch": 1.2233009708737863,
+      "grad_norm": 0.1783309429883957,
+      "learning_rate": 3.478335497469219e-05,
+      "loss": 1.053591012954712,
+      "mean_token_accuracy": 0.6859150379896164,
+      "num_tokens": 12257187.0,
+      "step": 126
+    },
+    {
+      "entropy": 1.0370903313159943,
+      "epoch": 1.233009708737864,
+      "grad_norm": 0.16358032822608948,
+      "learning_rate": 3.45361210273113e-05,
+      "loss": 1.037482738494873,
+      "mean_token_accuracy": 0.6902541518211365,
+      "num_tokens": 12354720.0,
+      "step": 127
+    },
+    {
+      "entropy": 1.0623970925807953,
+      "epoch": 1.2427184466019416,
+      "grad_norm": 0.17698562145233154,
+      "learning_rate": 3.42877907731548e-05,
+      "loss": 1.0513551235198975,
+      "mean_token_accuracy": 0.6855376213788986,
+      "num_tokens": 12452643.0,
+      "step": 128
+    },
+    {
+      "entropy": 1.0202428475022316,
+      "epoch": 1.2524271844660193,
+      "grad_norm": 0.16466407477855682,
+      "learning_rate": 3.403839276116199e-05,
+      "loss": 1.0054267644882202,
+      "mean_token_accuracy": 0.6959393694996834,
+      "num_tokens": 12549738.0,
+      "step": 129
+    },
+    {
+      "entropy": 1.0314210504293442,
+      "epoch": 1.262135922330097,
+      "grad_norm": 0.16708038747310638,
+      "learning_rate": 3.378795566302541e-05,
+      "loss": 1.0263310670852661,
+      "mean_token_accuracy": 0.6902486681938171,
+      "num_tokens": 12647010.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.065687894821167,
+      "epoch": 1.2718446601941746,
+      "grad_norm": 0.16275963187217712,
+      "learning_rate": 3.3536508269894724e-05,
+      "loss": 1.066178321838379,
+      "mean_token_accuracy": 0.6823260933160782,
+      "num_tokens": 12744989.0,
+      "step": 131
+    },
+    {
+      "entropy": 1.0025106966495514,
+      "epoch": 1.2815533980582523,
+      "grad_norm": 0.16108715534210205,
+      "learning_rate": 3.3284079489066725e-05,
+      "loss": 1.00234055519104,
+      "mean_token_accuracy": 0.6987117454409599,
+      "num_tokens": 12841473.0,
+      "step": 132
+    },
+    {
+      "entropy": 1.0151324942708015,
+      "epoch": 1.29126213592233,
+      "grad_norm": 0.18361398577690125,
+      "learning_rate": 3.303069834066206e-05,
+      "loss": 1.0149610042572021,
+      "mean_token_accuracy": 0.6954189836978912,
+      "num_tokens": 12938931.0,
+      "step": 133
+    },
+    {
+      "entropy": 1.0325570032000542,
+      "epoch": 1.3009708737864076,
+      "grad_norm": 0.16869449615478516,
+      "learning_rate": 3.2776393954289e-05,
+      "loss": 1.0201994180679321,
+      "mean_token_accuracy": 0.6910282000899315,
+      "num_tokens": 13036221.0,
+      "step": 134
+    },
+    {
+      "entropy": 1.042519137263298,
+      "epoch": 1.3106796116504853,
+      "grad_norm": 0.1671546846628189,
+      "learning_rate": 3.252119556569454e-05,
+      "loss": 1.0274970531463623,
+      "mean_token_accuracy": 0.689722865819931,
+      "num_tokens": 13133336.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.0281556621193886,
+      "epoch": 1.3203883495145632,
+      "grad_norm": 0.16985583305358887,
+      "learning_rate": 3.226513251340341e-05,
+      "loss": 1.0146019458770752,
+      "mean_token_accuracy": 0.6919778510928154,
+      "num_tokens": 13230747.0,
+      "step": 136
+    },
+    {
+      "entropy": 1.0338696613907814,
+      "epoch": 1.3300970873786409,
+      "grad_norm": 0.1647978276014328,
+      "learning_rate": 3.200823423534519e-05,
+      "loss": 1.0158432722091675,
+      "mean_token_accuracy": 0.6931767091155052,
+      "num_tokens": 13328746.0,
+      "step": 137
+    },
+    {
+      "entropy": 1.008351869881153,
+      "epoch": 1.3398058252427185,
+      "grad_norm": 0.16024157404899597,
+      "learning_rate": 3.175053026547002e-05,
+      "loss": 1.0044397115707397,
+      "mean_token_accuracy": 0.6963309571146965,
+      "num_tokens": 13425842.0,
+      "step": 138
+    },
+    {
+      "entropy": 0.9922654777765274,
+      "epoch": 1.3495145631067962,
+      "grad_norm": 0.1634853333234787,
+      "learning_rate": 3.149205023035324e-05,
+      "loss": 0.9884682893753052,
+      "mean_token_accuracy": 0.6995600908994675,
+      "num_tokens": 13522580.0,
+      "step": 139
+    },
+    {
+      "entropy": 1.0148945599794388,
+      "epoch": 1.3592233009708738,
+      "grad_norm": 0.17045436799526215,
+      "learning_rate": 3.123282384578947e-05,
+      "loss": 1.0163800716400146,
+      "mean_token_accuracy": 0.6940884217619896,
+      "num_tokens": 13620098.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.9754997342824936,
+      "epoch": 1.3689320388349515,
+      "grad_norm": 0.1622450202703476,
+      "learning_rate": 3.097288091337635e-05,
+      "loss": 0.9774127006530762,
+      "mean_token_accuracy": 0.7022812962532043,
+      "num_tokens": 13717872.0,
+      "step": 141
+    },
+    {
+      "entropy": 1.0089435130357742,
+      "epoch": 1.3786407766990292,
+      "grad_norm": 0.16076694428920746,
+      "learning_rate": 3.0712251317088424e-05,
+      "loss": 1.0084456205368042,
+      "mean_token_accuracy": 0.6960485577583313,
+      "num_tokens": 13814624.0,
+      "step": 142
+    },
+    {
+      "entropy": 1.035470925271511,
+      "epoch": 1.3883495145631068,
+      "grad_norm": 0.17493759095668793,
+      "learning_rate": 3.0450965019841592e-05,
+      "loss": 1.0213847160339355,
+      "mean_token_accuracy": 0.6898086741566658,
+      "num_tokens": 13912077.0,
+      "step": 143
+    },
+    {
+      "entropy": 1.0338619202375412,
+      "epoch": 1.3980582524271845,
+      "grad_norm": 0.18111379444599152,
+      "learning_rate": 3.018905206004846e-05,
+      "loss": 1.0271329879760742,
+      "mean_token_accuracy": 0.6900457143783569,
+      "num_tokens": 14009785.0,
+      "step": 144
+    },
+    {
+      "entropy": 1.02719896286726,
+      "epoch": 1.4077669902912622,
+      "grad_norm": 0.1971462070941925,
+      "learning_rate": 2.9926542548165e-05,
+      "loss": 1.0115890502929688,
+      "mean_token_accuracy": 0.6943076327443123,
+      "num_tokens": 14107396.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.0307827293872833,
+      "epoch": 1.4174757281553398,
+      "grad_norm": 0.16493792831897736,
+      "learning_rate": 2.9663466663228978e-05,
+      "loss": 1.0267215967178345,
+      "mean_token_accuracy": 0.6911192238330841,
+      "num_tokens": 14204726.0,
+      "step": 146
+    },
+    {
+      "entropy": 0.9859491810202599,
+      "epoch": 1.4271844660194175,
+      "grad_norm": 0.16403748095035553,
+      "learning_rate": 2.939985464939043e-05,
+      "loss": 0.9758182764053345,
+      "mean_token_accuracy": 0.7027716338634491,
+      "num_tokens": 14302498.0,
+      "step": 147
+    },
+    {
+      "entropy": 1.0136100724339485,
+      "epoch": 1.4368932038834952,
+      "grad_norm": 0.16826969385147095,
+      "learning_rate": 2.91357368124347e-05,
+      "loss": 1.019405722618103,
+      "mean_token_accuracy": 0.6951397210359573,
+      "num_tokens": 14400012.0,
+      "step": 148
+    },
+    {
+      "entropy": 1.0319994240999222,
+      "epoch": 1.4466019417475728,
+      "grad_norm": 0.20015913248062134,
+      "learning_rate": 2.887114351629839e-05,
+      "loss": 1.0211818218231201,
+      "mean_token_accuracy": 0.6888004466891289,
+      "num_tokens": 14497317.0,
+      "step": 149
+    },
+    {
+      "entropy": 1.0356996357440948,
+      "epoch": 1.4563106796116505,
+      "grad_norm": 0.16202102601528168,
+      "learning_rate": 2.8606105179578585e-05,
+      "loss": 1.0237884521484375,
+      "mean_token_accuracy": 0.6904987245798111,
+      "num_tokens": 14594267.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.0340648591518402,
+      "epoch": 1.4660194174757282,
+      "grad_norm": 0.17005588114261627,
+      "learning_rate": 2.834065227203584e-05,
+      "loss": 1.022267460823059,
+      "mean_token_accuracy": 0.6913619935512543,
+      "num_tokens": 14691722.0,
+      "step": 151
+    },
+    {
+      "entropy": 1.0183274075388908,
+      "epoch": 1.4757281553398058,
+      "grad_norm": 0.16634872555732727,
+      "learning_rate": 2.8074815311091263e-05,
+      "loss": 1.00758957862854,
+      "mean_token_accuracy": 0.6955769658088684,
+      "num_tokens": 14789029.0,
+      "step": 152
+    },
+    {
+      "entropy": 1.0224798768758774,
+      "epoch": 1.4854368932038835,
+      "grad_norm": 0.17028877139091492,
+      "learning_rate": 2.780862485831814e-05,
+      "loss": 1.0180184841156006,
+      "mean_token_accuracy": 0.6907658129930496,
+      "num_tokens": 14886881.0,
+      "step": 153
+    },
+    {
+      "entropy": 1.0085324943065643,
+      "epoch": 1.4951456310679612,
+      "grad_norm": 0.16488611698150635,
+      "learning_rate": 2.754211151592841e-05,
+      "loss": 1.0009567737579346,
+      "mean_token_accuracy": 0.6960463523864746,
+      "num_tokens": 14984847.0,
+      "step": 154
+    },
+    {
+      "entropy": 1.0101035311818123,
+      "epoch": 1.5048543689320388,
+      "grad_norm": 0.16247566044330597,
+      "learning_rate": 2.7275305923254606e-05,
+      "loss": 1.0088591575622559,
+      "mean_token_accuracy": 0.6951913386583328,
+      "num_tokens": 15082543.0,
+      "step": 155
+    },
+    {
+      "entropy": 0.9859557524323463,
+      "epoch": 1.5145631067961165,
+      "grad_norm": 0.16418179869651794,
+      "learning_rate": 2.7008238753227383e-05,
+      "loss": 0.987535834312439,
+      "mean_token_accuracy": 0.698513351380825,
+      "num_tokens": 15180431.0,
+      "step": 156
+    },
+    {
+      "entropy": 1.000753603875637,
+      "epoch": 1.5242718446601942,
+      "grad_norm": 0.17313377559185028,
+      "learning_rate": 2.674094070884926e-05,
+      "loss": 0.9962478876113892,
+      "mean_token_accuracy": 0.6992179900407791,
+      "num_tokens": 15277711.0,
+      "step": 157
+    },
+    {
+      "entropy": 1.023979589343071,
+      "epoch": 1.5339805825242718,
+      "grad_norm": 0.16977423429489136,
+      "learning_rate": 2.6473442519664933e-05,
+      "loss": 1.0200791358947754,
+      "mean_token_accuracy": 0.6923878714442253,
+      "num_tokens": 15374988.0,
+      "step": 158
+    },
+    {
+      "entropy": 1.0136018842458725,
+      "epoch": 1.5436893203883495,
+      "grad_norm": 0.17464333772659302,
+      "learning_rate": 2.6205774938228432e-05,
+      "loss": 1.000876784324646,
+      "mean_token_accuracy": 0.6955864802002907,
+      "num_tokens": 15472290.0,
+      "step": 159
+    },
+    {
+      "entropy": 0.981790617108345,
+      "epoch": 1.5533980582524272,
+      "grad_norm": 0.1634012758731842,
+      "learning_rate": 2.5937968736567746e-05,
+      "loss": 0.9761837720870972,
+      "mean_token_accuracy": 0.7053327187895775,
+      "num_tokens": 15569870.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.0178728476166725,
+      "epoch": 1.5631067961165048,
+      "grad_norm": 0.16406911611557007,
+      "learning_rate": 2.5670054702647146e-05,
+      "loss": 1.0168684720993042,
+      "mean_token_accuracy": 0.691955953836441,
+      "num_tokens": 15667339.0,
+      "step": 161
+    },
+    {
+      "entropy": 1.0100897997617722,
+      "epoch": 1.5728155339805825,
+      "grad_norm": 0.1631382256746292,
+      "learning_rate": 2.540206363682768e-05,
+      "loss": 1.0096473693847656,
+      "mean_token_accuracy": 0.6947467923164368,
+      "num_tokens": 15764247.0,
+      "step": 162
+    },
+    {
+      "entropy": 1.0208442211151123,
+      "epoch": 1.5825242718446602,
+      "grad_norm": 0.1662256121635437,
+      "learning_rate": 2.513402634832627e-05,
+      "loss": 1.014646053314209,
+      "mean_token_accuracy": 0.6943413838744164,
+      "num_tokens": 15861365.0,
+      "step": 163
+    },
+    {
+      "entropy": 1.0086032897233963,
+      "epoch": 1.5922330097087378,
+      "grad_norm": 0.16566972434520721,
+      "learning_rate": 2.486597365167374e-05,
+      "loss": 1.0061399936676025,
+      "mean_token_accuracy": 0.6969663575291634,
+      "num_tokens": 15957083.0,
+      "step": 164
+    },
+    {
+      "entropy": 1.0157904401421547,
+      "epoch": 1.6019417475728155,
+      "grad_norm": 0.19315719604492188,
+      "learning_rate": 2.4597936363172327e-05,
+      "loss": 1.0081403255462646,
+      "mean_token_accuracy": 0.6941899582743645,
+      "num_tokens": 16052280.0,
+      "step": 165
+    },
+    {
+      "entropy": 0.9923936128616333,
+      "epoch": 1.6116504854368932,
+      "grad_norm": 0.16755102574825287,
+      "learning_rate": 2.4329945297352856e-05,
+      "loss": 0.9879144430160522,
+      "mean_token_accuracy": 0.7004513517022133,
+      "num_tokens": 16149822.0,
+      "step": 166
+    },
+    {
+      "entropy": 1.019317552447319,
+      "epoch": 1.6213592233009708,
+      "grad_norm": 0.16500990092754364,
+      "learning_rate": 2.4062031263432267e-05,
+      "loss": 1.0097726583480835,
+      "mean_token_accuracy": 0.6913179829716682,
+      "num_tokens": 16246545.0,
+      "step": 167
+    },
+    {
+      "entropy": 1.0108042284846306,
+      "epoch": 1.6310679611650487,
+      "grad_norm": 0.1684969663619995,
+      "learning_rate": 2.379422506177157e-05,
+      "loss": 1.0046005249023438,
+      "mean_token_accuracy": 0.6959811896085739,
+      "num_tokens": 16344324.0,
+      "step": 168
+    },
+    {
+      "entropy": 1.014855496585369,
+      "epoch": 1.6407766990291264,
+      "grad_norm": 0.16363738477230072,
+      "learning_rate": 2.352655748033508e-05,
+      "loss": 1.0102152824401855,
+      "mean_token_accuracy": 0.6956550776958466,
+      "num_tokens": 16442411.0,
+      "step": 169
+    },
+    {
+      "entropy": 1.0056051313877106,
+      "epoch": 1.650485436893204,
+      "grad_norm": 0.17643426358699799,
+      "learning_rate": 2.3259059291150744e-05,
+      "loss": 1.0087074041366577,
+      "mean_token_accuracy": 0.6949120983481407,
+      "num_tokens": 16540618.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.9741454049944878,
+      "epoch": 1.6601941747572817,
+      "grad_norm": 0.16666218638420105,
+      "learning_rate": 2.2991761246772623e-05,
+      "loss": 0.9674159288406372,
+      "mean_token_accuracy": 0.7026141285896301,
+      "num_tokens": 16638399.0,
+      "step": 171
+    },
+    {
+      "entropy": 1.0255334451794624,
+      "epoch": 1.6699029126213594,
+      "grad_norm": 0.1657601147890091,
+      "learning_rate": 2.2724694076745396e-05,
+      "loss": 1.0157989263534546,
+      "mean_token_accuracy": 0.6930963471531868,
+      "num_tokens": 16735834.0,
+      "step": 172
+    },
+    {
+      "entropy": 1.0137359574437141,
+      "epoch": 1.679611650485437,
+      "grad_norm": 0.16463129222393036,
+      "learning_rate": 2.245788848407159e-05,
+      "loss": 1.0035295486450195,
+      "mean_token_accuracy": 0.696049340069294,
+      "num_tokens": 16832922.0,
+      "step": 173
+    },
+    {
+      "entropy": 1.0246630683541298,
+      "epoch": 1.6893203883495147,
+      "grad_norm": 0.1674775928258896,
+      "learning_rate": 2.2191375141681867e-05,
+      "loss": 1.0074610710144043,
+      "mean_token_accuracy": 0.6950727999210358,
+      "num_tokens": 16930339.0,
+      "step": 174
+    },
+    {
+      "entropy": 1.0009135901927948,
+      "epoch": 1.6990291262135924,
+      "grad_norm": 0.16697858273983002,
+      "learning_rate": 2.1925184688908733e-05,
+      "loss": 0.9845885038375854,
+      "mean_token_accuracy": 0.7005789056420326,
+      "num_tokens": 17026260.0,
+      "step": 175
+    },
+    {
+      "entropy": 0.9530843198299408,
+      "epoch": 1.70873786407767,
+      "grad_norm": 0.16688551008701324,
+      "learning_rate": 2.165934772796417e-05,
+      "loss": 0.942157506942749,
+      "mean_token_accuracy": 0.7122049182653427,
+      "num_tokens": 17123607.0,
+      "step": 176
+    },
+    {
+      "entropy": 1.0441362410783768,
+      "epoch": 1.7184466019417477,
+      "grad_norm": 0.1778694987297058,
+      "learning_rate": 2.139389482042142e-05,
+      "loss": 1.0378202199935913,
+      "mean_token_accuracy": 0.6889643222093582,
+      "num_tokens": 17221652.0,
+      "step": 177
+    },
+    {
+      "entropy": 1.0085002332925797,
+      "epoch": 1.7281553398058254,
+      "grad_norm": 0.17363892495632172,
+      "learning_rate": 2.1128856483701624e-05,
+      "loss": 1.0034621953964233,
+      "mean_token_accuracy": 0.6959685385227203,
+      "num_tokens": 17318735.0,
+      "step": 178
+    },
+    {
+      "entropy": 0.9722514152526855,
+      "epoch": 1.737864077669903,
+      "grad_norm": 0.15921640396118164,
+      "learning_rate": 2.0864263187565307e-05,
+      "loss": 0.9693209528923035,
+      "mean_token_accuracy": 0.7054861709475517,
+      "num_tokens": 17415256.0,
+      "step": 179
+    },
+    {
+      "entropy": 0.9984462335705757,
+      "epoch": 1.7475728155339807,
+      "grad_norm": 0.17000792920589447,
+      "learning_rate": 2.0600145350609586e-05,
+      "loss": 0.9936474561691284,
+      "mean_token_accuracy": 0.6967209428548813,
+      "num_tokens": 17511588.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.0166059508919716,
+      "epoch": 1.7572815533980584,
+      "grad_norm": 0.16876304149627686,
+      "learning_rate": 2.033653333677103e-05,
+      "loss": 1.0153465270996094,
+      "mean_token_accuracy": 0.6905340850353241,
+      "num_tokens": 17608711.0,
+      "step": 181
+    },
+    {
+      "entropy": 0.9773640930652618,
+      "epoch": 1.766990291262136,
+      "grad_norm": 0.1631699800491333,
+      "learning_rate": 2.0073457451835e-05,
+      "loss": 0.9742135405540466,
+      "mean_token_accuracy": 0.7022309750318527,
+      "num_tokens": 17706685.0,
+      "step": 182
+    },
+    {
+      "entropy": 0.977528765797615,
+      "epoch": 1.7766990291262137,
+      "grad_norm": 0.17157995700836182,
+      "learning_rate": 1.9810947939951546e-05,
+      "loss": 0.9730427265167236,
+      "mean_token_accuracy": 0.7017250508069992,
+      "num_tokens": 17803446.0,
+      "step": 183
+    },
+    {
+      "entropy": 1.0205107182264328,
+      "epoch": 1.7864077669902914,
+      "grad_norm": 0.16768775880336761,
+      "learning_rate": 1.9549034980158403e-05,
+      "loss": 1.0211429595947266,
+      "mean_token_accuracy": 0.6909284666180611,
+      "num_tokens": 17901149.0,
+      "step": 184
+    },
+    {
+      "entropy": 1.0032172948122025,
+      "epoch": 1.796116504854369,
+      "grad_norm": 0.17262859642505646,
+      "learning_rate": 1.928774868291158e-05,
+      "loss": 0.9938658475875854,
+      "mean_token_accuracy": 0.6984692439436913,
+      "num_tokens": 17998157.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.018993966281414,
+      "epoch": 1.8058252427184467,
+      "grad_norm": 0.16821308434009552,
+      "learning_rate": 1.9027119086623645e-05,
+      "loss": 1.01228928565979,
+      "mean_token_accuracy": 0.6922355368733406,
+      "num_tokens": 18095640.0,
+      "step": 186
+    },
+    {
+      "entropy": 1.0428634360432625,
+      "epoch": 1.8155339805825244,
+      "grad_norm": 0.1655871570110321,
+      "learning_rate": 1.8767176154210537e-05,
+      "loss": 1.0325472354888916,
+      "mean_token_accuracy": 0.6878891810774803,
+      "num_tokens": 18193453.0,
+      "step": 187
+    },
+    {
+      "entropy": 1.0023284628987312,
+      "epoch": 1.825242718446602,
+      "grad_norm": 0.16816659271717072,
+      "learning_rate": 1.850794976964677e-05,
+      "loss": 0.9961387515068054,
+      "mean_token_accuracy": 0.6975979804992676,
+      "num_tokens": 18291043.0,
+      "step": 188
+    },
+    {
+      "entropy": 1.0056410059332848,
+      "epoch": 1.8349514563106797,
+      "grad_norm": 0.1668543815612793,
+      "learning_rate": 1.8249469734529994e-05,
+      "loss": 0.9960080981254578,
+      "mean_token_accuracy": 0.6956286355853081,
+      "num_tokens": 18388297.0,
+      "step": 189
+    },
+    {
+      "entropy": 0.9780382961034775,
+      "epoch": 1.8446601941747574,
+      "grad_norm": 0.164035826921463,
+      "learning_rate": 1.7991765764654813e-05,
+      "loss": 0.9792324900627136,
+      "mean_token_accuracy": 0.7039750963449478,
+      "num_tokens": 18486092.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.9816233143210411,
+      "epoch": 1.854368932038835,
+      "grad_norm": 0.17571981251239777,
+      "learning_rate": 1.7734867486596594e-05,
+      "loss": 0.9816774725914001,
+      "mean_token_accuracy": 0.704063206911087,
+      "num_tokens": 18583836.0,
+      "step": 191
+    },
+    {
+      "entropy": 0.9972858354449272,
+      "epoch": 1.8640776699029127,
+      "grad_norm": 0.16646282374858856,
+      "learning_rate": 1.7478804434305465e-05,
+      "loss": 0.9942795634269714,
+      "mean_token_accuracy": 0.6969089061021805,
+      "num_tokens": 18681222.0,
+      "step": 192
+    },
+    {
+      "entropy": 1.0527583062648773,
+      "epoch": 1.8737864077669903,
+      "grad_norm": 0.18073324859142303,
+      "learning_rate": 1.7223606045711006e-05,
+      "loss": 1.041966199874878,
+      "mean_token_accuracy": 0.6850250214338303,
+      "num_tokens": 18778612.0,
+      "step": 193
+    },
+    {
+      "entropy": 1.0321912541985512,
+      "epoch": 1.883495145631068,
+      "grad_norm": 0.1724151074886322,
+      "learning_rate": 1.6969301659337944e-05,
+      "loss": 1.0273559093475342,
+      "mean_token_accuracy": 0.6907031312584877,
+      "num_tokens": 18875486.0,
+      "step": 194
+    },
+    {
+      "entropy": 1.0083372667431831,
+      "epoch": 1.8932038834951457,
+      "grad_norm": 0.16481448709964752,
+      "learning_rate": 1.6715920510933274e-05,
+      "loss": 0.9994081258773804,
+      "mean_token_accuracy": 0.6958953067660332,
+      "num_tokens": 18972930.0,
+      "step": 195
+    },
+    {
+      "entropy": 0.9780029803514481,
+      "epoch": 1.9029126213592233,
+      "grad_norm": 0.1704963743686676,
+      "learning_rate": 1.646349173010528e-05,
+      "loss": 0.956865668296814,
+      "mean_token_accuracy": 0.7070175260305405,
+      "num_tokens": 19069768.0,
+      "step": 196
+    },
+    {
+      "entropy": 0.9872947707772255,
+      "epoch": 1.912621359223301,
+      "grad_norm": 0.16887593269348145,
+      "learning_rate": 1.6212044336974596e-05,
+      "loss": 0.972030758857727,
+      "mean_token_accuracy": 0.7037985101342201,
+      "num_tokens": 19166951.0,
+      "step": 197
+    },
+    {
+      "entropy": 1.0222311094403267,
+      "epoch": 1.9223300970873787,
+      "grad_norm": 0.16921208798885345,
+      "learning_rate": 1.596160723883802e-05,
+      "loss": 1.0143109560012817,
+      "mean_token_accuracy": 0.6901704221963882,
+      "num_tokens": 19264885.0,
+      "step": 198
+    },
+    {
+      "entropy": 1.0139800533652306,
+      "epoch": 1.9320388349514563,
+      "grad_norm": 0.17927469313144684,
+      "learning_rate": 1.57122092268452e-05,
+      "loss": 1.0080219507217407,
+      "mean_token_accuracy": 0.6949594989418983,
+      "num_tokens": 19359705.0,
+      "step": 199
+    },
+    {
+      "entropy": 0.9707703441381454,
+      "epoch": 1.941747572815534,
+      "grad_norm": 0.1701960563659668,
+      "learning_rate": 1.5463878972688705e-05,
+      "loss": 0.9729690551757812,
+      "mean_token_accuracy": 0.7024904564023018,
+      "num_tokens": 19453554.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.9972376450896263,
+      "epoch": 1.9514563106796117,
+      "grad_norm": 0.1662243902683258,
+      "learning_rate": 1.5216645025307814e-05,
+      "loss": 1.0034092664718628,
+      "mean_token_accuracy": 0.6966458633542061,
+      "num_tokens": 19550972.0,
+      "step": 201
+    },
+    {
+      "entropy": 1.0318325906991959,
+      "epoch": 1.9611650485436893,
+      "grad_norm": 0.16855081915855408,
+      "learning_rate": 1.4970535807606453e-05,
+      "loss": 1.0335042476654053,
+      "mean_token_accuracy": 0.6879718452692032,
+      "num_tokens": 19648684.0,
+      "step": 202
+    },
+    {
+      "entropy": 1.040890358388424,
+      "epoch": 1.970873786407767,
+      "grad_norm": 0.1725233495235443,
+      "learning_rate": 1.4725579613185547e-05,
+      "loss": 1.047674536705017,
+      "mean_token_accuracy": 0.684162549674511,
+      "num_tokens": 19745486.0,
+      "step": 203
+    },
+    {
+      "entropy": 1.019149050116539,
+      "epoch": 1.9805825242718447,
+      "grad_norm": 0.36389368772506714,
+      "learning_rate": 1.4481804603090357e-05,
+      "loss": 1.0158822536468506,
+      "mean_token_accuracy": 0.6921994760632515,
+      "num_tokens": 19842391.0,
+      "step": 204
+    },
+    {
+      "entropy": 0.9926181361079216,
+      "epoch": 1.9902912621359223,
+      "grad_norm": 0.16377584636211395,
+      "learning_rate": 1.4239238802572908e-05,
+      "loss": 0.9823788404464722,
+      "mean_token_accuracy": 0.7014920264482498,
+      "num_tokens": 19939763.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.0035971254110336,
+      "epoch": 2.0,
+      "grad_norm": 0.17520146071910858,
+      "learning_rate": 1.3997910097870165e-05,
+      "loss": 0.9908577799797058,
+      "mean_token_accuracy": 0.6976755559444427,
+      "num_tokens": 20037788.0,
+      "step": 206
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 309,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.818590877692199e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}