Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-70B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-70B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-70B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "up_proj",
+    "q_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7224a008c60902ccb6baace88dafd8aa6d6c4530bf7afb02b715a1e87e7d25a
+size 3313653480

chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

trainer_state.json

@@ -0,0 +1,3154 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 312,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4476254880428314,
+      "epoch": 0.009685230024213076,
+      "grad_norm": 1.3588989973068237,
+      "learning_rate": 0.0,
+      "loss": 2.2350821495056152,
+      "mean_token_accuracy": 0.5031501688063145,
+      "num_tokens": 96574.0,
+      "step": 1
+    },
+    {
+      "entropy": 1.4007079899311066,
+      "epoch": 0.01937046004842615,
+      "grad_norm": 1.2849924564361572,
+      "learning_rate": 3.125e-06,
+      "loss": 2.14509654045105,
+      "mean_token_accuracy": 0.5205462127923965,
+      "num_tokens": 191520.0,
+      "step": 2
+    },
+    {
+      "entropy": 1.4708476215600967,
+      "epoch": 0.029055690072639227,
+      "grad_norm": 1.2314231395721436,
+      "learning_rate": 6.25e-06,
+      "loss": 2.158306121826172,
+      "mean_token_accuracy": 0.5159582570195198,
+      "num_tokens": 287631.0,
+      "step": 3
+    },
+    {
+      "entropy": 1.5651142746210098,
+      "epoch": 0.0387409200968523,
+      "grad_norm": 0.9785442352294922,
+      "learning_rate": 9.375000000000001e-06,
+      "loss": 2.0755105018615723,
+      "mean_token_accuracy": 0.5181002020835876,
+      "num_tokens": 384739.0,
+      "step": 4
+    },
+    {
+      "entropy": 1.579958975315094,
+      "epoch": 0.048426150121065374,
+      "grad_norm": 0.7276659607887268,
+      "learning_rate": 1.25e-05,
+      "loss": 1.930289626121521,
+      "mean_token_accuracy": 0.5408441498875618,
+      "num_tokens": 481674.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.8359138667583466,
+      "epoch": 0.05811138014527845,
+      "grad_norm": 0.5072433352470398,
+      "learning_rate": 1.5625e-05,
+      "loss": 1.9374589920043945,
+      "mean_token_accuracy": 0.5299954265356064,
+      "num_tokens": 576090.0,
+      "step": 6
+    },
+    {
+      "entropy": 1.8671298921108246,
+      "epoch": 0.06779661016949153,
+      "grad_norm": 0.44714638590812683,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 1.8663867712020874,
+      "mean_token_accuracy": 0.5436736792325974,
+      "num_tokens": 673951.0,
+      "step": 7
+    },
+    {
+      "entropy": 1.933446392416954,
+      "epoch": 0.0774818401937046,
+      "grad_norm": 0.5136867761611938,
+      "learning_rate": 2.1875e-05,
+      "loss": 1.7743638753890991,
+      "mean_token_accuracy": 0.5584963485598564,
+      "num_tokens": 771899.0,
+      "step": 8
+    },
+    {
+      "entropy": 1.9850385338068008,
+      "epoch": 0.08716707021791767,
+      "grad_norm": 0.5442814826965332,
+      "learning_rate": 2.5e-05,
+      "loss": 1.7758773565292358,
+      "mean_token_accuracy": 0.5573008134961128,
+      "num_tokens": 868206.0,
+      "step": 9
+    },
+    {
+      "entropy": 1.9424707889556885,
+      "epoch": 0.09685230024213075,
+      "grad_norm": 0.5084043741226196,
+      "learning_rate": 2.8125000000000003e-05,
+      "loss": 1.7394293546676636,
+      "mean_token_accuracy": 0.5608929991722107,
+      "num_tokens": 964219.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.8063177168369293,
+      "epoch": 0.10653753026634383,
+      "grad_norm": 0.3967217206954956,
+      "learning_rate": 3.125e-05,
+      "loss": 1.6807222366333008,
+      "mean_token_accuracy": 0.5679514929652214,
+      "num_tokens": 1062125.0,
+      "step": 11
+    },
+    {
+      "entropy": 1.638314574956894,
+      "epoch": 0.1162227602905569,
+      "grad_norm": 0.5107802152633667,
+      "learning_rate": 3.4375e-05,
+      "loss": 1.6123173236846924,
+      "mean_token_accuracy": 0.583977460861206,
+      "num_tokens": 1159319.0,
+      "step": 12
+    },
+    {
+      "entropy": 1.5576547533273697,
+      "epoch": 0.12590799031476999,
+      "grad_norm": 0.3253207802772522,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 1.6140292882919312,
+      "mean_token_accuracy": 0.5774664506316185,
+      "num_tokens": 1256881.0,
+      "step": 13
+    },
+    {
+      "entropy": 1.4602049440145493,
+      "epoch": 0.13559322033898305,
+      "grad_norm": 0.3317743241786957,
+      "learning_rate": 4.0625000000000005e-05,
+      "loss": 1.5576837062835693,
+      "mean_token_accuracy": 0.586665965616703,
+      "num_tokens": 1351620.0,
+      "step": 14
+    },
+    {
+      "entropy": 1.4226532131433487,
+      "epoch": 0.14527845036319612,
+      "grad_norm": 0.2991933524608612,
+      "learning_rate": 4.375e-05,
+      "loss": 1.5325368642807007,
+      "mean_token_accuracy": 0.5934261903166771,
+      "num_tokens": 1449026.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.453083336353302,
+      "epoch": 0.1549636803874092,
+      "grad_norm": 0.26260727643966675,
+      "learning_rate": 4.6875e-05,
+      "loss": 1.5117604732513428,
+      "mean_token_accuracy": 0.5941561385989189,
+      "num_tokens": 1546355.0,
+      "step": 16
+    },
+    {
+      "entropy": 1.4769755005836487,
+      "epoch": 0.16464891041162227,
+      "grad_norm": 0.24356529116630554,
+      "learning_rate": 5e-05,
+      "loss": 1.5000947713851929,
+      "mean_token_accuracy": 0.5963850915431976,
+      "num_tokens": 1644308.0,
+      "step": 17
+    },
+    {
+      "entropy": 1.4608993381261826,
+      "epoch": 0.17433414043583534,
+      "grad_norm": 0.27541783452033997,
+      "learning_rate": 4.999859193643945e-05,
+      "loss": 1.4296932220458984,
+      "mean_token_accuracy": 0.6135347038507462,
+      "num_tokens": 1741625.0,
+      "step": 18
+    },
+    {
+      "entropy": 1.4715198874473572,
+      "epoch": 0.18401937046004843,
+      "grad_norm": 0.24254612624645233,
+      "learning_rate": 4.999436790436924e-05,
+      "loss": 1.4109866619110107,
+      "mean_token_accuracy": 0.6162872388958931,
+      "num_tokens": 1839077.0,
+      "step": 19
+    },
+    {
+      "entropy": 1.4909484386444092,
+      "epoch": 0.1937046004842615,
+      "grad_norm": 0.24249622225761414,
+      "learning_rate": 4.9987328379605816e-05,
+      "loss": 1.4181320667266846,
+      "mean_token_accuracy": 0.6092976555228233,
+      "num_tokens": 1936754.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.468568578362465,
+      "epoch": 0.2033898305084746,
+      "grad_norm": 0.21892383694648743,
+      "learning_rate": 4.9977474155117045e-05,
+      "loss": 1.4207193851470947,
+      "mean_token_accuracy": 0.6143926084041595,
+      "num_tokens": 2034049.0,
+      "step": 21
+    },
+    {
+      "entropy": 1.4144536703824997,
+      "epoch": 0.21307506053268765,
+      "grad_norm": 0.24802649021148682,
+      "learning_rate": 4.9964806340932865e-05,
+      "loss": 1.3823838233947754,
+      "mean_token_accuracy": 0.6196955218911171,
+      "num_tokens": 2131193.0,
+      "step": 22
+    },
+    {
+      "entropy": 1.4027946889400482,
+      "epoch": 0.22276029055690072,
+      "grad_norm": 0.2114405333995819,
+      "learning_rate": 4.9949326364020314e-05,
+      "loss": 1.3788299560546875,
+      "mean_token_accuracy": 0.6205098181962967,
+      "num_tokens": 2228554.0,
+      "step": 23
+    },
+    {
+      "entropy": 1.3630587011575699,
+      "epoch": 0.2324455205811138,
+      "grad_norm": 0.18567043542861938,
+      "learning_rate": 4.993103596812268e-05,
+      "loss": 1.3441742658615112,
+      "mean_token_accuracy": 0.6255069524049759,
+      "num_tokens": 2325761.0,
+      "step": 24
+    },
+    {
+      "entropy": 1.3504581600427628,
+      "epoch": 0.24213075060532688,
+      "grad_norm": 0.18140752613544464,
+      "learning_rate": 4.9909937213563165e-05,
+      "loss": 1.3367418050765991,
+      "mean_token_accuracy": 0.6294924318790436,
+      "num_tokens": 2420230.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.3556479513645172,
+      "epoch": 0.25181598062953997,
+      "grad_norm": 0.19952309131622314,
+      "learning_rate": 4.988603247701277e-05,
+      "loss": 1.329459547996521,
+      "mean_token_accuracy": 0.6290309652686119,
+      "num_tokens": 2515756.0,
+      "step": 26
+    },
+    {
+      "entropy": 1.3174784928560257,
+      "epoch": 0.26150121065375304,
+      "grad_norm": 0.2040238082408905,
+      "learning_rate": 4.985932445122257e-05,
+      "loss": 1.3146334886550903,
+      "mean_token_accuracy": 0.6350805461406708,
+      "num_tokens": 2611239.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.337596833705902,
+      "epoch": 0.2711864406779661,
+      "grad_norm": 0.1924348771572113,
+      "learning_rate": 4.982981614472039e-05,
+      "loss": 1.325057864189148,
+      "mean_token_accuracy": 0.6268098577857018,
+      "num_tokens": 2708463.0,
+      "step": 28
+    },
+    {
+      "entropy": 1.295740321278572,
+      "epoch": 0.28087167070217917,
+      "grad_norm": 0.18018221855163574,
+      "learning_rate": 4.979751088147192e-05,
+      "loss": 1.2943141460418701,
+      "mean_token_accuracy": 0.6371493935585022,
+      "num_tokens": 2806105.0,
+      "step": 29
+    },
+    {
+      "entropy": 1.2824859917163849,
+      "epoch": 0.29055690072639223,
+      "grad_norm": 0.1904715597629547,
+      "learning_rate": 4.97624123005063e-05,
+      "loss": 1.282959222793579,
+      "mean_token_accuracy": 0.6380399465560913,
+      "num_tokens": 2903815.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.274334877729416,
+      "epoch": 0.30024213075060535,
+      "grad_norm": 0.17454974353313446,
+      "learning_rate": 4.972452435550614e-05,
+      "loss": 1.2828162908554077,
+      "mean_token_accuracy": 0.6363882124423981,
+      "num_tokens": 3000656.0,
+      "step": 31
+    },
+    {
+      "entropy": 1.298419788479805,
+      "epoch": 0.3099273607748184,
+      "grad_norm": 0.16989658772945404,
+      "learning_rate": 4.968385131436222e-05,
+      "loss": 1.283800482749939,
+      "mean_token_accuracy": 0.6383039951324463,
+      "num_tokens": 3098537.0,
+      "step": 32
+    },
+    {
+      "entropy": 1.3138496726751328,
+      "epoch": 0.3196125907990315,
+      "grad_norm": 0.1963842809200287,
+      "learning_rate": 4.9640397758692715e-05,
+      "loss": 1.287667989730835,
+      "mean_token_accuracy": 0.6366405189037323,
+      "num_tokens": 3195923.0,
+      "step": 33
+    },
+    {
+      "entropy": 1.316459745168686,
+      "epoch": 0.32929782082324455,
+      "grad_norm": 0.17209197580814362,
+      "learning_rate": 4.9594168583327094e-05,
+      "loss": 1.2872288227081299,
+      "mean_token_accuracy": 0.6377927586436272,
+      "num_tokens": 3293424.0,
+      "step": 34
+    },
+    {
+      "entropy": 1.2688211798667908,
+      "epoch": 0.3389830508474576,
+      "grad_norm": 0.1710345298051834,
+      "learning_rate": 4.954516899575473e-05,
+      "loss": 1.2326550483703613,
+      "mean_token_accuracy": 0.6454800367355347,
+      "num_tokens": 3390314.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.275676041841507,
+      "epoch": 0.3486682808716707,
+      "grad_norm": 0.16850367188453674,
+      "learning_rate": 4.949340451553833e-05,
+      "loss": 1.2480475902557373,
+      "mean_token_accuracy": 0.644576296210289,
+      "num_tokens": 3485697.0,
+      "step": 36
+    },
+    {
+      "entropy": 1.2488422840833664,
+      "epoch": 0.3583535108958838,
+      "grad_norm": 0.17575675249099731,
+      "learning_rate": 4.943888097369216e-05,
+      "loss": 1.2449251413345337,
+      "mean_token_accuracy": 0.6482485681772232,
+      "num_tokens": 3583116.0,
+      "step": 37
+    },
+    {
+      "entropy": 1.20504991710186,
+      "epoch": 0.36803874092009686,
+      "grad_norm": 0.16634824872016907,
+      "learning_rate": 4.938160451202521e-05,
+      "loss": 1.210385799407959,
+      "mean_token_accuracy": 0.6547724008560181,
+      "num_tokens": 3678517.0,
+      "step": 38
+    },
+    {
+      "entropy": 1.236429587006569,
+      "epoch": 0.37772397094430993,
+      "grad_norm": 0.15360407531261444,
+      "learning_rate": 4.9321581582449365e-05,
+      "loss": 1.2314069271087646,
+      "mean_token_accuracy": 0.6504408493638039,
+      "num_tokens": 3775690.0,
+      "step": 39
+    },
+    {
+      "entropy": 1.2558289468288422,
+      "epoch": 0.387409200968523,
+      "grad_norm": 0.1626509577035904,
+      "learning_rate": 4.9258818946252624e-05,
+      "loss": 1.253812551498413,
+      "mean_token_accuracy": 0.6433829963207245,
+      "num_tokens": 3873239.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.2188849300146103,
+      "epoch": 0.39709443099273606,
+      "grad_norm": 0.16926692426204681,
+      "learning_rate": 4.9193323673337476e-05,
+      "loss": 1.2143971920013428,
+      "mean_token_accuracy": 0.6519596204161644,
+      "num_tokens": 3970507.0,
+      "step": 41
+    },
+    {
+      "entropy": 1.2296934127807617,
+      "epoch": 0.4067796610169492,
+      "grad_norm": 0.17094968259334564,
+      "learning_rate": 4.912510314142448e-05,
+      "loss": 1.2195372581481934,
+      "mean_token_accuracy": 0.6513471901416779,
+      "num_tokens": 4067145.0,
+      "step": 42
+    },
+    {
+      "entropy": 1.2168416231870651,
+      "epoch": 0.41646489104116224,
+      "grad_norm": 0.15936042368412018,
+      "learning_rate": 4.9054165035221236e-05,
+      "loss": 1.2123249769210815,
+      "mean_token_accuracy": 0.6524514406919479,
+      "num_tokens": 4165113.0,
+      "step": 43
+    },
+    {
+      "entropy": 1.1896271854639053,
+      "epoch": 0.4261501210653753,
+      "grad_norm": 0.15422837436199188,
+      "learning_rate": 4.898051734555676e-05,
+      "loss": 1.1957529783248901,
+      "mean_token_accuracy": 0.6586505621671677,
+      "num_tokens": 4262663.0,
+      "step": 44
+    },
+    {
+      "entropy": 1.2026645243167877,
+      "epoch": 0.4358353510895884,
+      "grad_norm": 0.16097040474414825,
+      "learning_rate": 4.890416836848127e-05,
+      "loss": 1.1992558240890503,
+      "mean_token_accuracy": 0.6544606015086174,
+      "num_tokens": 4359900.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.1843629777431488,
+      "epoch": 0.44552058111380144,
+      "grad_norm": 0.1597595363855362,
+      "learning_rate": 4.88251267043318e-05,
+      "loss": 1.1683578491210938,
+      "mean_token_accuracy": 0.6638541519641876,
+      "num_tokens": 4456985.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.1679482907056808,
+      "epoch": 0.4552058111380145,
+      "grad_norm": 0.15203599631786346,
+      "learning_rate": 4.87434012567633e-05,
+      "loss": 1.1561784744262695,
+      "mean_token_accuracy": 0.6650881245732307,
+      "num_tokens": 4554634.0,
+      "step": 47
+    },
+    {
+      "entropy": 1.2105818390846252,
+      "epoch": 0.4648910411622276,
+      "grad_norm": 0.15597011148929596,
+      "learning_rate": 4.865900123174574e-05,
+      "loss": 1.1950294971466064,
+      "mean_token_accuracy": 0.6544563546776772,
+      "num_tokens": 4652249.0,
+      "step": 48
+    },
+    {
+      "entropy": 1.2046389728784561,
+      "epoch": 0.4745762711864407,
+      "grad_norm": 0.15933600068092346,
+      "learning_rate": 4.857193613652711e-05,
+      "loss": 1.183128833770752,
+      "mean_token_accuracy": 0.6564416959881783,
+      "num_tokens": 4748832.0,
+      "step": 49
+    },
+    {
+      "entropy": 1.2113090008497238,
+      "epoch": 0.48426150121065376,
+      "grad_norm": 0.15904678404331207,
+      "learning_rate": 4.848221577856243e-05,
+      "loss": 1.1915383338928223,
+      "mean_token_accuracy": 0.6567506939172745,
+      "num_tokens": 4845986.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1754019558429718,
+      "epoch": 0.4939467312348668,
+      "grad_norm": 0.15548165142536163,
+      "learning_rate": 4.838985026440905e-05,
+      "loss": 1.1645618677139282,
+      "mean_token_accuracy": 0.6654537692666054,
+      "num_tokens": 4942777.0,
+      "step": 51
+    },
+    {
+      "entropy": 1.1717564463615417,
+      "epoch": 0.5036319612590799,
+      "grad_norm": 0.16495537757873535,
+      "learning_rate": 4.829484999858815e-05,
+      "loss": 1.1674176454544067,
+      "mean_token_accuracy": 0.659358873963356,
+      "num_tokens": 5040326.0,
+      "step": 52
+    },
+    {
+      "entropy": 1.1637324541807175,
+      "epoch": 0.513317191283293,
+      "grad_norm": 0.16075873374938965,
+      "learning_rate": 4.819722568241274e-05,
+      "loss": 1.1621849536895752,
+      "mean_token_accuracy": 0.663130484521389,
+      "num_tokens": 5137838.0,
+      "step": 53
+    },
+    {
+      "entropy": 1.1703068912029266,
+      "epoch": 0.5230024213075061,
+      "grad_norm": 0.16048873960971832,
+      "learning_rate": 4.8096988312782174e-05,
+      "loss": 1.1667448282241821,
+      "mean_token_accuracy": 0.6606730446219444,
+      "num_tokens": 5234578.0,
+      "step": 54
+    },
+    {
+      "entropy": 1.1869463622570038,
+      "epoch": 0.5326876513317191,
+      "grad_norm": 0.1574316769838333,
+      "learning_rate": 4.799414918094347e-05,
+      "loss": 1.1867117881774902,
+      "mean_token_accuracy": 0.6562071144580841,
+      "num_tokens": 5332439.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.1833796501159668,
+      "epoch": 0.5423728813559322,
+      "grad_norm": 0.1627049446105957,
+      "learning_rate": 4.788871987121937e-05,
+      "loss": 1.1771697998046875,
+      "mean_token_accuracy": 0.6603464111685753,
+      "num_tokens": 5430102.0,
+      "step": 56
+    },
+    {
+      "entropy": 1.1720363795757294,
+      "epoch": 0.5520581113801453,
+      "grad_norm": 0.30378302931785583,
+      "learning_rate": 4.77807122597034e-05,
+      "loss": 1.1528563499450684,
+      "mean_token_accuracy": 0.6617102548480034,
+      "num_tokens": 5528264.0,
+      "step": 57
+    },
+    {
+      "entropy": 1.1582888960838318,
+      "epoch": 0.5617433414043583,
+      "grad_norm": 0.1613432615995407,
+      "learning_rate": 4.767013851292212e-05,
+      "loss": 1.1482983827590942,
+      "mean_token_accuracy": 0.6675111055374146,
+      "num_tokens": 5625491.0,
+      "step": 58
+    },
+    {
+      "entropy": 1.1704413145780563,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.15665417909622192,
+      "learning_rate": 4.755701108646463e-05,
+      "loss": 1.1476081609725952,
+      "mean_token_accuracy": 0.6660670340061188,
+      "num_tokens": 5723563.0,
+      "step": 59
+    },
+    {
+      "entropy": 1.1684788465499878,
+      "epoch": 0.5811138014527845,
+      "grad_norm": 0.1587826907634735,
+      "learning_rate": 4.744134272357948e-05,
+      "loss": 1.15497624874115,
+      "mean_token_accuracy": 0.662791483104229,
+      "num_tokens": 5821106.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1705324202775955,
+      "epoch": 0.5907990314769975,
+      "grad_norm": 0.1650344729423523,
+      "learning_rate": 4.732314645373921e-05,
+      "loss": 1.168868899345398,
+      "mean_token_accuracy": 0.659059427678585,
+      "num_tokens": 5918772.0,
+      "step": 61
+    },
+    {
+      "entropy": 1.1340993344783783,
+      "epoch": 0.6004842615012107,
+      "grad_norm": 0.15585103631019592,
+      "learning_rate": 4.7202435591172676e-05,
+      "loss": 1.1181915998458862,
+      "mean_token_accuracy": 0.6716931834816933,
+      "num_tokens": 6016758.0,
+      "step": 62
+    },
+    {
+      "entropy": 1.1135689467191696,
+      "epoch": 0.6101694915254238,
+      "grad_norm": 0.15927211940288544,
+      "learning_rate": 4.7079223733365237e-05,
+      "loss": 1.1100869178771973,
+      "mean_token_accuracy": 0.6781439930200577,
+      "num_tokens": 6113210.0,
+      "step": 63
+    },
+    {
+      "entropy": 1.1656545102596283,
+      "epoch": 0.6198547215496368,
+      "grad_norm": 0.1585935652256012,
+      "learning_rate": 4.6953524759527054e-05,
+      "loss": 1.1522451639175415,
+      "mean_token_accuracy": 0.664880745112896,
+      "num_tokens": 6210272.0,
+      "step": 64
+    },
+    {
+      "entropy": 1.1241111010313034,
+      "epoch": 0.6295399515738499,
+      "grad_norm": 0.15607106685638428,
+      "learning_rate": 4.6825352829029705e-05,
+      "loss": 1.1200485229492188,
+      "mean_token_accuracy": 0.671033151447773,
+      "num_tokens": 6307273.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.135048821568489,
+      "epoch": 0.639225181598063,
+      "grad_norm": 0.1598539501428604,
+      "learning_rate": 4.6694722379811185e-05,
+      "loss": 1.1302683353424072,
+      "mean_token_accuracy": 0.6700672060251236,
+      "num_tokens": 6404865.0,
+      "step": 66
+    },
+    {
+      "entropy": 1.1544559001922607,
+      "epoch": 0.648910411622276,
+      "grad_norm": 0.16644105315208435,
+      "learning_rate": 4.656164812674951e-05,
+      "loss": 1.1450002193450928,
+      "mean_token_accuracy": 0.6640813648700714,
+      "num_tokens": 6501650.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.132233053445816,
+      "epoch": 0.6585956416464891,
+      "grad_norm": 0.16045193374156952,
+      "learning_rate": 4.642614506000523e-05,
+      "loss": 1.121936559677124,
+      "mean_token_accuracy": 0.6716863736510277,
+      "num_tokens": 6599028.0,
+      "step": 68
+    },
+    {
+      "entropy": 1.1332774758338928,
+      "epoch": 0.6682808716707022,
+      "grad_norm": 0.16355212032794952,
+      "learning_rate": 4.628822844333278e-05,
+      "loss": 1.1154568195343018,
+      "mean_token_accuracy": 0.6706111207604408,
+      "num_tokens": 6695913.0,
+      "step": 69
+    },
+    {
+      "entropy": 1.117134228348732,
+      "epoch": 0.6779661016949152,
+      "grad_norm": 0.1609818935394287,
+      "learning_rate": 4.614791381236115e-05,
+      "loss": 1.108933925628662,
+      "mean_token_accuracy": 0.6739098057150841,
+      "num_tokens": 6793755.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.123718187212944,
+      "epoch": 0.6876513317191283,
+      "grad_norm": 0.16711370646953583,
+      "learning_rate": 4.6005216972843864e-05,
+      "loss": 1.1205769777297974,
+      "mean_token_accuracy": 0.6695492789149284,
+      "num_tokens": 6889869.0,
+      "step": 71
+    },
+    {
+      "entropy": 1.1330177187919617,
+      "epoch": 0.6973365617433414,
+      "grad_norm": 0.15766654908657074,
+      "learning_rate": 4.5860153998878494e-05,
+      "loss": 1.136279582977295,
+      "mean_token_accuracy": 0.6664912849664688,
+      "num_tokens": 6987242.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.1289773285388947,
+      "epoch": 0.7070217917675545,
+      "grad_norm": 0.167424738407135,
+      "learning_rate": 4.571274123109606e-05,
+      "loss": 1.1257307529449463,
+      "mean_token_accuracy": 0.6683443114161491,
+      "num_tokens": 7083683.0,
+      "step": 73
+    },
+    {
+      "entropy": 1.1089464724063873,
+      "epoch": 0.7167070217917676,
+      "grad_norm": 0.29150310158729553,
+      "learning_rate": 4.5562995274820284e-05,
+      "loss": 1.0986303091049194,
+      "mean_token_accuracy": 0.6785000637173653,
+      "num_tokens": 7180007.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.1168130189180374,
+      "epoch": 0.7263922518159807,
+      "grad_norm": 0.1623908281326294,
+      "learning_rate": 4.541093299819714e-05,
+      "loss": 1.1109479665756226,
+      "mean_token_accuracy": 0.6751653999090195,
+      "num_tokens": 7276880.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.1238265335559845,
+      "epoch": 0.7360774818401937,
+      "grad_norm": 0.16212861239910126,
+      "learning_rate": 4.5256571530294664e-05,
+      "loss": 1.104738473892212,
+      "mean_token_accuracy": 0.6755917444825172,
+      "num_tokens": 7374051.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.1079749763011932,
+      "epoch": 0.7457627118644068,
+      "grad_norm": 0.15648601949214935,
+      "learning_rate": 4.5099928259173516e-05,
+      "loss": 1.1000884771347046,
+      "mean_token_accuracy": 0.6749309673905373,
+      "num_tokens": 7471906.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.12865149974823,
+      "epoch": 0.7554479418886199,
+      "grad_norm": 0.15941032767295837,
+      "learning_rate": 4.4941020829928245e-05,
+      "loss": 1.1193838119506836,
+      "mean_token_accuracy": 0.6708522811532021,
+      "num_tokens": 7569981.0,
+      "step": 78
+    },
+    {
+      "entropy": 1.1111949235200882,
+      "epoch": 0.7651331719128329,
+      "grad_norm": 0.16523951292037964,
+      "learning_rate": 4.477986714269972e-05,
+      "loss": 1.1221985816955566,
+      "mean_token_accuracy": 0.6717454493045807,
+      "num_tokens": 7663566.0,
+      "step": 79
+    },
+    {
+      "entropy": 1.100884273648262,
+      "epoch": 0.774818401937046,
+      "grad_norm": 0.16432514786720276,
+      "learning_rate": 4.4616485350658685e-05,
+      "loss": 1.104296088218689,
+      "mean_token_accuracy": 0.6767150536179543,
+      "num_tokens": 7761120.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.1229810565710068,
+      "epoch": 0.784503631961259,
+      "grad_norm": 0.17101751267910004,
+      "learning_rate": 4.445089385796099e-05,
+      "loss": 1.1202152967453003,
+      "mean_token_accuracy": 0.6708082109689713,
+      "num_tokens": 7859307.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.134630709886551,
+      "epoch": 0.7941888619854721,
+      "grad_norm": 0.1734648197889328,
+      "learning_rate": 4.4283111317674374e-05,
+      "loss": 1.1201170682907104,
+      "mean_token_accuracy": 0.6716241240501404,
+      "num_tokens": 7956863.0,
+      "step": 82
+    },
+    {
+      "entropy": 1.1259192377328873,
+      "epoch": 0.8038740920096852,
+      "grad_norm": 0.16475871205329895,
+      "learning_rate": 4.4113156629677316e-05,
+      "loss": 1.1068096160888672,
+      "mean_token_accuracy": 0.674552284181118,
+      "num_tokens": 8053100.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.0983240008354187,
+      "epoch": 0.8135593220338984,
+      "grad_norm": 0.17292368412017822,
+      "learning_rate": 4.394104893853007e-05,
+      "loss": 1.0763158798217773,
+      "mean_token_accuracy": 0.6806622296571732,
+      "num_tokens": 8151004.0,
+      "step": 84
+    },
+    {
+      "entropy": 1.1288987547159195,
+      "epoch": 0.8232445520581114,
+      "grad_norm": 0.16203227639198303,
+      "learning_rate": 4.3766807631318106e-05,
+      "loss": 1.1195769309997559,
+      "mean_token_accuracy": 0.671344555914402,
+      "num_tokens": 8248668.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.1410782486200333,
+      "epoch": 0.8329297820823245,
+      "grad_norm": 0.16533830761909485,
+      "learning_rate": 4.359045233546827e-05,
+      "loss": 1.1256823539733887,
+      "mean_token_accuracy": 0.6698467060923576,
+      "num_tokens": 8345497.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.0999096632003784,
+      "epoch": 0.8426150121065376,
+      "grad_norm": 0.17075778543949127,
+      "learning_rate": 4.341200291653781e-05,
+      "loss": 1.0872827768325806,
+      "mean_token_accuracy": 0.6762653961777687,
+      "num_tokens": 8443405.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.0918122231960297,
+      "epoch": 0.8523002421307506,
+      "grad_norm": 0.1702580451965332,
+      "learning_rate": 4.323147947597666e-05,
+      "loss": 1.0806095600128174,
+      "mean_token_accuracy": 0.6800353080034256,
+      "num_tokens": 8540868.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.0941280871629715,
+      "epoch": 0.8619854721549637,
+      "grad_norm": 0.16791851818561554,
+      "learning_rate": 4.3048902348863116e-05,
+      "loss": 1.0981943607330322,
+      "mean_token_accuracy": 0.6758224442601204,
+      "num_tokens": 8638186.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.0951679199934006,
+      "epoch": 0.8716707021791767,
+      "grad_norm": 0.1697986125946045,
+      "learning_rate": 4.2864292101613136e-05,
+      "loss": 1.0987718105316162,
+      "mean_token_accuracy": 0.6747072339057922,
+      "num_tokens": 8735354.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1223643571138382,
+      "epoch": 0.8813559322033898,
+      "grad_norm": 0.2108999490737915,
+      "learning_rate": 4.267766952966369e-05,
+      "loss": 1.1219862699508667,
+      "mean_token_accuracy": 0.6695899069309235,
+      "num_tokens": 8832051.0,
+      "step": 91
+    },
+    {
+      "entropy": 1.1099307239055634,
+      "epoch": 0.8910411622276029,
+      "grad_norm": 0.18003661930561066,
+      "learning_rate": 4.248905565513023e-05,
+      "loss": 1.0990370512008667,
+      "mean_token_accuracy": 0.6750800833106041,
+      "num_tokens": 8929471.0,
+      "step": 92
+    },
+    {
+      "entropy": 1.1097175031900406,
+      "epoch": 0.9007263922518159,
+      "grad_norm": 0.18473289906978607,
+      "learning_rate": 4.229847172443866e-05,
+      "loss": 1.0890154838562012,
+      "mean_token_accuracy": 0.6764732897281647,
+      "num_tokens": 9026545.0,
+      "step": 93
+    },
+    {
+      "entropy": 1.1048423945903778,
+      "epoch": 0.910411622276029,
+      "grad_norm": 0.17208412289619446,
+      "learning_rate": 4.210593920593201e-05,
+      "loss": 1.1022264957427979,
+      "mean_token_accuracy": 0.6741294264793396,
+      "num_tokens": 9124261.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.0943758487701416,
+      "epoch": 0.9200968523002422,
+      "grad_norm": 0.18163490295410156,
+      "learning_rate": 4.191147978745218e-05,
+      "loss": 1.0901988744735718,
+      "mean_token_accuracy": 0.6788006126880646,
+      "num_tokens": 9220673.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.1178465783596039,
+      "epoch": 0.9297820823244553,
+      "grad_norm": 0.1731623411178589,
+      "learning_rate": 4.171511537389684e-05,
+      "loss": 1.1111767292022705,
+      "mean_token_accuracy": 0.6736986339092255,
+      "num_tokens": 9318321.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.0975633561611176,
+      "epoch": 0.9394673123486683,
+      "grad_norm": 0.18208175897598267,
+      "learning_rate": 4.151686808475204e-05,
+      "loss": 1.0915802717208862,
+      "mean_token_accuracy": 0.6765245422720909,
+      "num_tokens": 9415394.0,
+      "step": 97
+    },
+    {
+      "entropy": 1.114044964313507,
+      "epoch": 0.9491525423728814,
+      "grad_norm": 0.17058847844600677,
+      "learning_rate": 4.131676025160047e-05,
+      "loss": 1.1074469089508057,
+      "mean_token_accuracy": 0.6720306798815727,
+      "num_tokens": 9512880.0,
+      "step": 98
+    },
+    {
+      "entropy": 1.07672019302845,
+      "epoch": 0.9588377723970944,
+      "grad_norm": 0.18478929996490479,
+      "learning_rate": 4.111481441560598e-05,
+      "loss": 1.0674448013305664,
+      "mean_token_accuracy": 0.6838654279708862,
+      "num_tokens": 9606053.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.0935327857732773,
+      "epoch": 0.9685230024213075,
+      "grad_norm": 0.17510797083377838,
+      "learning_rate": 4.091105332497439e-05,
+      "loss": 1.0954351425170898,
+      "mean_token_accuracy": 0.6775561571121216,
+      "num_tokens": 9702569.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.0855822414159775,
+      "epoch": 0.9782082324455206,
+      "grad_norm": 0.18510136008262634,
+      "learning_rate": 4.070549993239106e-05,
+      "loss": 1.0844038724899292,
+      "mean_token_accuracy": 0.681103028357029,
+      "num_tokens": 9799976.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.1072624325752258,
+      "epoch": 0.9878934624697336,
+      "grad_norm": 0.2109905481338501,
+      "learning_rate": 4.049817739243532e-05,
+      "loss": 1.1046338081359863,
+      "mean_token_accuracy": 0.6760703399777412,
+      "num_tokens": 9895575.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.0761565268039703,
+      "epoch": 0.9975786924939467,
+      "grad_norm": 0.18559105694293976,
+      "learning_rate": 4.028910905897229e-05,
+      "loss": 1.0674762725830078,
+      "mean_token_accuracy": 0.6826195642352104,
+      "num_tokens": 9992394.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.1065305471420288,
+      "epoch": 1.0,
+      "grad_norm": 0.3376278877258301,
+      "learning_rate": 4.007831848252211e-05,
+      "loss": 1.09256911277771,
+      "mean_token_accuracy": 0.678756445646286,
+      "num_tokens": 10016415.0,
+      "step": 104
+    },
+    {
+      "entropy": 1.1107248812913895,
+      "epoch": 1.0096852300242132,
+      "grad_norm": 0.1708710938692093,
+      "learning_rate": 3.986582940760717e-05,
+      "loss": 1.0717049837112427,
+      "mean_token_accuracy": 0.6800583377480507,
+      "num_tokens": 10113793.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.0776572525501251,
+      "epoch": 1.0193704600484261,
+      "grad_norm": 0.17748922109603882,
+      "learning_rate": 3.965166577007733e-05,
+      "loss": 1.0392060279846191,
+      "mean_token_accuracy": 0.6892436370253563,
+      "num_tokens": 10211074.0,
+      "step": 106
+    },
+    {
+      "entropy": 1.0571173429489136,
+      "epoch": 1.0290556900726393,
+      "grad_norm": 0.18791398406028748,
+      "learning_rate": 3.94358516944137e-05,
+      "loss": 1.038546085357666,
+      "mean_token_accuracy": 0.6879089176654816,
+      "num_tokens": 10307987.0,
+      "step": 107
+    },
+    {
+      "entropy": 1.0400206819176674,
+      "epoch": 1.0387409200968523,
+      "grad_norm": 0.16869165003299713,
+      "learning_rate": 3.9218411491011176e-05,
+      "loss": 1.0324063301086426,
+      "mean_token_accuracy": 0.6877342015504837,
+      "num_tokens": 10405148.0,
+      "step": 108
+    },
+    {
+      "entropy": 1.0476296544075012,
+      "epoch": 1.0484261501210654,
+      "grad_norm": 0.18576034903526306,
+      "learning_rate": 3.899936965343989e-05,
+      "loss": 1.0663259029388428,
+      "mean_token_accuracy": 0.6821715012192726,
+      "num_tokens": 10502344.0,
+      "step": 109
+    },
+    {
+      "entropy": 0.9960673823952675,
+      "epoch": 1.0581113801452784,
+      "grad_norm": 0.17832568287849426,
+      "learning_rate": 3.877875085568623e-05,
+      "loss": 1.0000097751617432,
+      "mean_token_accuracy": 0.6989128738641739,
+      "num_tokens": 10599599.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.0352009981870651,
+      "epoch": 1.0677966101694916,
+      "grad_norm": 0.17328514158725739,
+      "learning_rate": 3.8556579949373384e-05,
+      "loss": 1.0506186485290527,
+      "mean_token_accuracy": 0.6840933337807655,
+      "num_tokens": 10694645.0,
+      "step": 111
+    },
+    {
+      "entropy": 1.0331084430217743,
+      "epoch": 1.0774818401937045,
+      "grad_norm": 0.18105825781822205,
+      "learning_rate": 3.833288196096194e-05,
+      "loss": 1.0261142253875732,
+      "mean_token_accuracy": 0.6913967728614807,
+      "num_tokens": 10790067.0,
+      "step": 112
+    },
+    {
+      "entropy": 1.0757844671607018,
+      "epoch": 1.0871670702179177,
+      "grad_norm": 0.1806410551071167,
+      "learning_rate": 3.8107682088930794e-05,
+      "loss": 1.0621310472488403,
+      "mean_token_accuracy": 0.6843468695878983,
+      "num_tokens": 10885655.0,
+      "step": 113
+    },
+    {
+      "entropy": 1.0726244300603867,
+      "epoch": 1.0968523002421307,
+      "grad_norm": 0.17602035403251648,
+      "learning_rate": 3.7881005700938635e-05,
+      "loss": 1.0586791038513184,
+      "mean_token_accuracy": 0.6825271546840668,
+      "num_tokens": 10983568.0,
+      "step": 114
+    },
+    {
+      "entropy": 1.0513464957475662,
+      "epoch": 1.1065375302663438,
+      "grad_norm": 0.1740947663784027,
+      "learning_rate": 3.7652878330966415e-05,
+      "loss": 1.035830020904541,
+      "mean_token_accuracy": 0.6891666278243065,
+      "num_tokens": 11081209.0,
+      "step": 115
+    },
+    {
+      "entropy": 1.0637426376342773,
+      "epoch": 1.116222760290557,
+      "grad_norm": 0.1756405085325241,
+      "learning_rate": 3.742332567644107e-05,
+      "loss": 1.0529723167419434,
+      "mean_token_accuracy": 0.6864708811044693,
+      "num_tokens": 11178554.0,
+      "step": 116
+    },
+    {
+      "entropy": 1.032620556652546,
+      "epoch": 1.12590799031477,
+      "grad_norm": 0.1746027022600174,
+      "learning_rate": 3.719237359534087e-05,
+      "loss": 1.0131734609603882,
+      "mean_token_accuracy": 0.6930168569087982,
+      "num_tokens": 11275773.0,
+      "step": 117
+    },
+    {
+      "entropy": 1.0357091426849365,
+      "epoch": 1.1355932203389831,
+      "grad_norm": 0.17611640691757202,
+      "learning_rate": 3.6960048103282566e-05,
+      "loss": 1.0305650234222412,
+      "mean_token_accuracy": 0.6928873062133789,
+      "num_tokens": 11372312.0,
+      "step": 118
+    },
+    {
+      "entropy": 1.0247548967599869,
+      "epoch": 1.145278450363196,
+      "grad_norm": 0.16733205318450928,
+      "learning_rate": 3.672637537059093e-05,
+      "loss": 1.0267465114593506,
+      "mean_token_accuracy": 0.6941879317164421,
+      "num_tokens": 11469895.0,
+      "step": 119
+    },
+    {
+      "entropy": 1.0328287035226822,
+      "epoch": 1.1549636803874093,
+      "grad_norm": 0.17758148908615112,
+      "learning_rate": 3.6491381719350756e-05,
+      "loss": 1.038550853729248,
+      "mean_token_accuracy": 0.687534786760807,
+      "num_tokens": 11567819.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.0606635063886642,
+      "epoch": 1.1646489104116222,
+      "grad_norm": 0.18219131231307983,
+      "learning_rate": 3.6255093620441834e-05,
+      "loss": 1.0544192790985107,
+      "mean_token_accuracy": 0.6846267804503441,
+      "num_tokens": 11665799.0,
+      "step": 121
+    },
+    {
+      "entropy": 1.0355539470911026,
+      "epoch": 1.1743341404358354,
+      "grad_norm": 0.18732059001922607,
+      "learning_rate": 3.6017537690557115e-05,
+      "loss": 1.0224016904830933,
+      "mean_token_accuracy": 0.6915983185172081,
+      "num_tokens": 11762356.0,
+      "step": 122
+    },
+    {
+      "entropy": 1.085768237709999,
+      "epoch": 1.1840193704600483,
+      "grad_norm": 0.20525461435317993,
+      "learning_rate": 3.577874068920446e-05,
+      "loss": 1.057420253753662,
+      "mean_token_accuracy": 0.6817465797066689,
+      "num_tokens": 11857832.0,
+      "step": 123
+    },
+    {
+      "entropy": 1.024247646331787,
+      "epoch": 1.1937046004842615,
+      "grad_norm": 0.17486801743507385,
+      "learning_rate": 3.553872951569236e-05,
+      "loss": 1.007400393486023,
+      "mean_token_accuracy": 0.6967773362994194,
+      "num_tokens": 11953563.0,
+      "step": 124
+    },
+    {
+      "entropy": 1.0304122641682625,
+      "epoch": 1.2033898305084745,
+      "grad_norm": 0.17227725684642792,
+      "learning_rate": 3.529753120609982e-05,
+      "loss": 1.0166981220245361,
+      "mean_token_accuracy": 0.6900071352720261,
+      "num_tokens": 12051630.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.0403276681900024,
+      "epoch": 1.2130750605326877,
+      "grad_norm": 0.17845618724822998,
+      "learning_rate": 3.505517293023088e-05,
+      "loss": 1.0411816835403442,
+      "mean_token_accuracy": 0.6878266781568527,
+      "num_tokens": 12149525.0,
+      "step": 126
+    },
+    {
+      "entropy": 1.0372891202569008,
+      "epoch": 1.2227602905569008,
+      "grad_norm": 0.17896883189678192,
+      "learning_rate": 3.481168198855409e-05,
+      "loss": 1.0509088039398193,
+      "mean_token_accuracy": 0.6861589029431343,
+      "num_tokens": 12246944.0,
+      "step": 127
+    },
+    {
+      "entropy": 1.0567465275526047,
+      "epoch": 1.2324455205811138,
+      "grad_norm": 0.1735103726387024,
+      "learning_rate": 3.456708580912725e-05,
+      "loss": 1.048382043838501,
+      "mean_token_accuracy": 0.6845468729734421,
+      "num_tokens": 12343964.0,
+      "step": 128
+    },
+    {
+      "entropy": 1.044258013367653,
+      "epoch": 1.242130750605327,
+      "grad_norm": 0.179740771651268,
+      "learning_rate": 3.432141194450772e-05,
+      "loss": 1.0318509340286255,
+      "mean_token_accuracy": 0.6892603412270546,
+      "num_tokens": 12441245.0,
+      "step": 129
+    },
+    {
+      "entropy": 1.0394302904605865,
+      "epoch": 1.25181598062954,
+      "grad_norm": 0.18108829855918884,
+      "learning_rate": 3.407468806864883e-05,
+      "loss": 1.0396722555160522,
+      "mean_token_accuracy": 0.6859570071101189,
+      "num_tokens": 12538938.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.0346315652132034,
+      "epoch": 1.261501210653753,
+      "grad_norm": 0.16831137239933014,
+      "learning_rate": 3.382694197378252e-05,
+      "loss": 1.0301259756088257,
+      "mean_token_accuracy": 0.6876077353954315,
+      "num_tokens": 12636750.0,
+      "step": 131
+    },
+    {
+      "entropy": 1.0501351803541183,
+      "epoch": 1.271186440677966,
+      "grad_norm": 0.18215049803256989,
+      "learning_rate": 3.357820156728867e-05,
+      "loss": 1.0379700660705566,
+      "mean_token_accuracy": 0.6864259541034698,
+      "num_tokens": 12732510.0,
+      "step": 132
+    },
+    {
+      "entropy": 1.056677907705307,
+      "epoch": 1.2808716707021792,
+      "grad_norm": 0.17387507855892181,
+      "learning_rate": 3.332849486855144e-05,
+      "loss": 1.0349537134170532,
+      "mean_token_accuracy": 0.6882437765598297,
+      "num_tokens": 12828202.0,
+      "step": 133
+    },
+    {
+      "entropy": 1.0694306939840317,
+      "epoch": 1.2905569007263922,
+      "grad_norm": 0.18142169713974,
+      "learning_rate": 3.307785000580313e-05,
+      "loss": 1.0491408109664917,
+      "mean_token_accuracy": 0.6856276169419289,
+      "num_tokens": 12924452.0,
+      "step": 134
+    },
+    {
+      "entropy": 1.0445284843444824,
+      "epoch": 1.3002421307506054,
+      "grad_norm": 0.17338253557682037,
+      "learning_rate": 3.282629521295556e-05,
+      "loss": 1.0387568473815918,
+      "mean_token_accuracy": 0.6878345832228661,
+      "num_tokens": 13021226.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.03611271828413,
+      "epoch": 1.3099273607748185,
+      "grad_norm": 0.17574281990528107,
+      "learning_rate": 3.257385882641971e-05,
+      "loss": 1.032334566116333,
+      "mean_token_accuracy": 0.6887728422880173,
+      "num_tokens": 13117072.0,
+      "step": 136
+    },
+    {
+      "entropy": 1.0164377763867378,
+      "epoch": 1.3196125907990315,
+      "grad_norm": 0.17293879389762878,
+      "learning_rate": 3.232056928191376e-05,
+      "loss": 1.0038666725158691,
+      "mean_token_accuracy": 0.6958679780364037,
+      "num_tokens": 13214927.0,
+      "step": 137
+    },
+    {
+      "entropy": 1.02716813236475,
+      "epoch": 1.3292978208232444,
+      "grad_norm": 0.1768789142370224,
+      "learning_rate": 3.2066455111259945e-05,
+      "loss": 1.0230392217636108,
+      "mean_token_accuracy": 0.6910659521818161,
+      "num_tokens": 13312250.0,
+      "step": 138
+    },
+    {
+      "entropy": 1.0242123305797577,
+      "epoch": 1.3389830508474576,
+      "grad_norm": 0.18655547499656677,
+      "learning_rate": 3.1811544939170575e-05,
+      "loss": 1.027778148651123,
+      "mean_token_accuracy": 0.6896005198359489,
+      "num_tokens": 13409125.0,
+      "step": 139
+    },
+    {
+      "entropy": 1.0247902870178223,
+      "epoch": 1.3486682808716708,
+      "grad_norm": 0.1732546091079712,
+      "learning_rate": 3.155586748002362e-05,
+      "loss": 1.0247126817703247,
+      "mean_token_accuracy": 0.6925645992159843,
+      "num_tokens": 13505714.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.0391114205121994,
+      "epoch": 1.3583535108958837,
+      "grad_norm": 0.1730506271123886,
+      "learning_rate": 3.1299451534628135e-05,
+      "loss": 1.0413163900375366,
+      "mean_token_accuracy": 0.6873403564095497,
+      "num_tokens": 13603282.0,
+      "step": 141
+    },
+    {
+      "entropy": 1.0580826699733734,
+      "epoch": 1.368038740920097,
+      "grad_norm": 0.17985138297080994,
+      "learning_rate": 3.1042325986980064e-05,
+      "loss": 1.048020362854004,
+      "mean_token_accuracy": 0.6840562298893929,
+      "num_tokens": 13699120.0,
+      "step": 142
+    },
+    {
+      "entropy": 1.034823827445507,
+      "epoch": 1.3777239709443099,
+      "grad_norm": 0.1742866039276123,
+      "learning_rate": 3.0784519801008546e-05,
+      "loss": 1.019952416419983,
+      "mean_token_accuracy": 0.6919118836522102,
+      "num_tokens": 13796639.0,
+      "step": 143
+    },
+    {
+      "entropy": 1.034243457019329,
+      "epoch": 1.387409200968523,
+      "grad_norm": 0.18104857206344604,
+      "learning_rate": 3.0526062017313254e-05,
+      "loss": 1.0257322788238525,
+      "mean_token_accuracy": 0.693074606359005,
+      "num_tokens": 13893968.0,
+      "step": 144
+    },
+    {
+      "entropy": 1.0477502346038818,
+      "epoch": 1.397094430992736,
+      "grad_norm": 0.17592455446720123,
+      "learning_rate": 3.0266981749893157e-05,
+      "loss": 1.0342764854431152,
+      "mean_token_accuracy": 0.6866420730948448,
+      "num_tokens": 13990892.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.0479690209031105,
+      "epoch": 1.4067796610169492,
+      "grad_norm": 0.1820760816335678,
+      "learning_rate": 3.0007308182866985e-05,
+      "loss": 1.0341622829437256,
+      "mean_token_accuracy": 0.687219500541687,
+      "num_tokens": 14087344.0,
+      "step": 146
+    },
+    {
+      "entropy": 1.0348835960030556,
+      "epoch": 1.4164648910411621,
+      "grad_norm": 0.1742199808359146,
+      "learning_rate": 2.974707056718571e-05,
+      "loss": 1.0271401405334473,
+      "mean_token_accuracy": 0.6896429732441902,
+      "num_tokens": 14185593.0,
+      "step": 147
+    },
+    {
+      "entropy": 1.0333365574479103,
+      "epoch": 1.4261501210653753,
+      "grad_norm": 0.177699476480484,
+      "learning_rate": 2.9486298217337637e-05,
+      "loss": 1.0313661098480225,
+      "mean_token_accuracy": 0.6886806860566139,
+      "num_tokens": 14283694.0,
+      "step": 148
+    },
+    {
+      "entropy": 1.0181496515870094,
+      "epoch": 1.4358353510895885,
+      "grad_norm": 0.17134729027748108,
+      "learning_rate": 2.9225020508046232e-05,
+      "loss": 1.009653091430664,
+      "mean_token_accuracy": 0.6931144669651985,
+      "num_tokens": 14380604.0,
+      "step": 149
+    },
+    {
+      "entropy": 1.038355827331543,
+      "epoch": 1.4455205811138014,
+      "grad_norm": 0.1845579445362091,
+      "learning_rate": 2.8963266870961226e-05,
+      "loss": 1.0307377576828003,
+      "mean_token_accuracy": 0.6884006261825562,
+      "num_tokens": 14478185.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.0334223732352257,
+      "epoch": 1.4552058111380144,
+      "grad_norm": 0.24978552758693695,
+      "learning_rate": 2.8701066791343288e-05,
+      "loss": 1.0366319417953491,
+      "mean_token_accuracy": 0.6860118135809898,
+      "num_tokens": 14575277.0,
+      "step": 151
+    },
+    {
+      "entropy": 1.0341607704758644,
+      "epoch": 1.4648910411622276,
+      "grad_norm": 0.17872564494609833,
+      "learning_rate": 2.8438449804742628e-05,
+      "loss": 1.0180070400238037,
+      "mean_token_accuracy": 0.6917398646473885,
+      "num_tokens": 14672534.0,
+      "step": 152
+    },
+    {
+      "entropy": 1.0590864270925522,
+      "epoch": 1.4745762711864407,
+      "grad_norm": 0.17723426222801208,
+      "learning_rate": 2.8175445493671972e-05,
+      "loss": 1.051386833190918,
+      "mean_token_accuracy": 0.6834967508912086,
+      "num_tokens": 14770518.0,
+      "step": 153
+    },
+    {
+      "entropy": 1.0175726786255836,
+      "epoch": 1.4842615012106537,
+      "grad_norm": 0.17911297082901,
+      "learning_rate": 2.7912083484274266e-05,
+      "loss": 0.9993636012077332,
+      "mean_token_accuracy": 0.6968463584780693,
+      "num_tokens": 14867845.0,
+      "step": 154
+    },
+    {
+      "entropy": 1.01533143222332,
+      "epoch": 1.4939467312348669,
+      "grad_norm": 0.17201802134513855,
+      "learning_rate": 2.7648393442985403e-05,
+      "loss": 0.9994634389877319,
+      "mean_token_accuracy": 0.6991591900587082,
+      "num_tokens": 14965851.0,
+      "step": 155
+    },
+    {
+      "entropy": 1.0423510372638702,
+      "epoch": 1.5036319612590798,
+      "grad_norm": 0.17345097661018372,
+      "learning_rate": 2.7384405073192454e-05,
+      "loss": 1.0389484167099,
+      "mean_token_accuracy": 0.6870414391160011,
+      "num_tokens": 15063279.0,
+      "step": 156
+    },
+    {
+      "entropy": 1.0268240571022034,
+      "epoch": 1.513317191283293,
+      "grad_norm": 0.18508543074131012,
+      "learning_rate": 2.7120148111887732e-05,
+      "loss": 1.029033899307251,
+      "mean_token_accuracy": 0.6905829161405563,
+      "num_tokens": 15160245.0,
+      "step": 157
+    },
+    {
+      "entropy": 1.0137058570981026,
+      "epoch": 1.5230024213075062,
+      "grad_norm": 0.1771853119134903,
+      "learning_rate": 2.6855652326319063e-05,
+      "loss": 1.011698603630066,
+      "mean_token_accuracy": 0.6954887434840202,
+      "num_tokens": 15257132.0,
+      "step": 158
+    },
+    {
+      "entropy": 1.0114770829677582,
+      "epoch": 1.5326876513317191,
+      "grad_norm": 0.1829320192337036,
+      "learning_rate": 2.659094751063666e-05,
+      "loss": 1.0108840465545654,
+      "mean_token_accuracy": 0.6935675665736198,
+      "num_tokens": 15354561.0,
+      "step": 159
+    },
+    {
+      "entropy": 1.0002705082297325,
+      "epoch": 1.542372881355932,
+      "grad_norm": 0.17417006194591522,
+      "learning_rate": 2.6326063482536934e-05,
+      "loss": 0.9976310729980469,
+      "mean_token_accuracy": 0.6973941773176193,
+      "num_tokens": 15451774.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.0380558297038078,
+      "epoch": 1.5520581113801453,
+      "grad_norm": 0.18505249917507172,
+      "learning_rate": 2.606103007990371e-05,
+      "loss": 1.027144193649292,
+      "mean_token_accuracy": 0.6899437606334686,
+      "num_tokens": 15545203.0,
+      "step": 161
+    },
+    {
+      "entropy": 1.0247594714164734,
+      "epoch": 1.5617433414043584,
+      "grad_norm": 0.17371951043605804,
+      "learning_rate": 2.579587715744712e-05,
+      "loss": 1.0144855976104736,
+      "mean_token_accuracy": 0.6931553333997726,
+      "num_tokens": 15642907.0,
+      "step": 162
+    },
+    {
+      "entropy": 1.025797888636589,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 0.17915304005146027,
+      "learning_rate": 2.5530634583340592e-05,
+      "loss": 1.014986276626587,
+      "mean_token_accuracy": 0.6926708817481995,
+      "num_tokens": 15739378.0,
+      "step": 163
+    },
+    {
+      "entropy": 1.0104742795228958,
+      "epoch": 1.5811138014527844,
+      "grad_norm": 0.17889732122421265,
+      "learning_rate": 2.526533223585641e-05,
+      "loss": 1.0031661987304688,
+      "mean_token_accuracy": 0.6982422098517418,
+      "num_tokens": 15836007.0,
+      "step": 164
+    },
+    {
+      "entropy": 1.0167404934763908,
+      "epoch": 1.5907990314769975,
+      "grad_norm": 0.1768142729997635,
+      "learning_rate": 2.5e-05,
+      "loss": 1.009070634841919,
+      "mean_token_accuracy": 0.6938488930463791,
+      "num_tokens": 15933286.0,
+      "step": 165
+    },
+    {
+      "entropy": 0.9942386075854301,
+      "epoch": 1.6004842615012107,
+      "grad_norm": 0.2674974203109741,
+      "learning_rate": 2.47346677641436e-05,
+      "loss": 0.9868994951248169,
+      "mean_token_accuracy": 0.7019253075122833,
+      "num_tokens": 16030240.0,
+      "step": 166
+    },
+    {
+      "entropy": 1.0080910325050354,
+      "epoch": 1.6101694915254239,
+      "grad_norm": 0.17362943291664124,
+      "learning_rate": 2.446936541665941e-05,
+      "loss": 1.0053434371948242,
+      "mean_token_accuracy": 0.6974755153059959,
+      "num_tokens": 16127744.0,
+      "step": 167
+    },
+    {
+      "entropy": 1.0292630195617676,
+      "epoch": 1.6198547215496368,
+      "grad_norm": 0.18086425960063934,
+      "learning_rate": 2.4204122842552896e-05,
+      "loss": 1.0237219333648682,
+      "mean_token_accuracy": 0.69173414260149,
+      "num_tokens": 16225217.0,
+      "step": 168
+    },
+    {
+      "entropy": 1.0308879762887955,
+      "epoch": 1.6295399515738498,
+      "grad_norm": 0.18374203145503998,
+      "learning_rate": 2.39389699200963e-05,
+      "loss": 1.0210704803466797,
+      "mean_token_accuracy": 0.6903956010937691,
+      "num_tokens": 16322913.0,
+      "step": 169
+    },
+    {
+      "entropy": 1.026255913078785,
+      "epoch": 1.639225181598063,
+      "grad_norm": 0.18278539180755615,
+      "learning_rate": 2.3673936517463075e-05,
+      "loss": 1.0169858932495117,
+      "mean_token_accuracy": 0.6928274035453796,
+      "num_tokens": 16419439.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.9919314235448837,
+      "epoch": 1.6489104116222761,
+      "grad_norm": 0.17901423573493958,
+      "learning_rate": 2.3409052489363342e-05,
+      "loss": 0.9918413162231445,
+      "mean_token_accuracy": 0.6992172300815582,
+      "num_tokens": 16516410.0,
+      "step": 171
+    },
+    {
+      "entropy": 1.0355833768844604,
+      "epoch": 1.658595641646489,
+      "grad_norm": 0.17641407251358032,
+      "learning_rate": 2.3144347673680936e-05,
+      "loss": 1.0381958484649658,
+      "mean_token_accuracy": 0.686020016670227,
+      "num_tokens": 16614080.0,
+      "step": 172
+    },
+    {
+      "entropy": 1.0149222612380981,
+      "epoch": 1.668280871670702,
+      "grad_norm": 0.17820627987384796,
+      "learning_rate": 2.287985188811228e-05,
+      "loss": 1.0130999088287354,
+      "mean_token_accuracy": 0.6923095658421516,
+      "num_tokens": 16711804.0,
+      "step": 173
+    },
+    {
+      "entropy": 1.0243929624557495,
+      "epoch": 1.6779661016949152,
+      "grad_norm": 0.18051660060882568,
+      "learning_rate": 2.261559492680755e-05,
+      "loss": 1.0211886167526245,
+      "mean_token_accuracy": 0.6929004341363907,
+      "num_tokens": 16808390.0,
+      "step": 174
+    },
+    {
+      "entropy": 1.008925937116146,
+      "epoch": 1.6876513317191284,
+      "grad_norm": 0.17846113443374634,
+      "learning_rate": 2.23516065570146e-05,
+      "loss": 0.9855537414550781,
+      "mean_token_accuracy": 0.7024274170398712,
+      "num_tokens": 16905846.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.0306098610162735,
+      "epoch": 1.6973365617433414,
+      "grad_norm": 0.17899896204471588,
+      "learning_rate": 2.2087916515725737e-05,
+      "loss": 1.0127032995224,
+      "mean_token_accuracy": 0.6937521025538445,
+      "num_tokens": 17002576.0,
+      "step": 176
+    },
+    {
+      "entropy": 1.0314870774745941,
+      "epoch": 1.7070217917675545,
+      "grad_norm": 0.1775345355272293,
+      "learning_rate": 2.182455450632803e-05,
+      "loss": 1.014747142791748,
+      "mean_token_accuracy": 0.6926901116967201,
+      "num_tokens": 17099808.0,
+      "step": 177
+    },
+    {
+      "entropy": 0.988661415874958,
+      "epoch": 1.7167070217917675,
+      "grad_norm": 0.18358361721038818,
+      "learning_rate": 2.1561550195257378e-05,
+      "loss": 0.9753472805023193,
+      "mean_token_accuracy": 0.7034848108887672,
+      "num_tokens": 17197523.0,
+      "step": 178
+    },
+    {
+      "entropy": 1.0072592198848724,
+      "epoch": 1.7263922518159807,
+      "grad_norm": 0.18197070062160492,
+      "learning_rate": 2.1298933208656718e-05,
+      "loss": 1.0025818347930908,
+      "mean_token_accuracy": 0.6967148259282112,
+      "num_tokens": 17292582.0,
+      "step": 179
+    },
+    {
+      "entropy": 1.0190014764666557,
+      "epoch": 1.7360774818401938,
+      "grad_norm": 0.1827293038368225,
+      "learning_rate": 2.103673312903878e-05,
+      "loss": 1.025465726852417,
+      "mean_token_accuracy": 0.691127598285675,
+      "num_tokens": 17389678.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.0355056896805763,
+      "epoch": 1.7457627118644068,
+      "grad_norm": 0.19289104640483856,
+      "learning_rate": 2.0774979491953777e-05,
+      "loss": 1.030361533164978,
+      "mean_token_accuracy": 0.6898203268647194,
+      "num_tokens": 17484545.0,
+      "step": 181
+    },
+    {
+      "entropy": 1.0205051228404045,
+      "epoch": 1.7554479418886197,
+      "grad_norm": 0.18466636538505554,
+      "learning_rate": 2.0513701782662366e-05,
+      "loss": 1.01216721534729,
+      "mean_token_accuracy": 0.6929588541388512,
+      "num_tokens": 17579500.0,
+      "step": 182
+    },
+    {
+      "entropy": 0.9954008832573891,
+      "epoch": 1.765133171912833,
+      "grad_norm": 0.18636250495910645,
+      "learning_rate": 2.025292943281429e-05,
+      "loss": 0.9910628199577332,
+      "mean_token_accuracy": 0.6983159705996513,
+      "num_tokens": 17674413.0,
+      "step": 183
+    },
+    {
+      "entropy": 1.00897516310215,
+      "epoch": 1.774818401937046,
+      "grad_norm": 0.1883116513490677,
+      "learning_rate": 1.9992691817133024e-05,
+      "loss": 1.0008057355880737,
+      "mean_token_accuracy": 0.6949929445981979,
+      "num_tokens": 17772206.0,
+      "step": 184
+    },
+    {
+      "entropy": 1.0539861172437668,
+      "epoch": 1.784503631961259,
+      "grad_norm": 0.188166081905365,
+      "learning_rate": 1.973301825010685e-05,
+      "loss": 1.0534381866455078,
+      "mean_token_accuracy": 0.6841985285282135,
+      "num_tokens": 17869518.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.0237281695008278,
+      "epoch": 1.794188861985472,
+      "grad_norm": 0.18438467383384705,
+      "learning_rate": 1.947393798268676e-05,
+      "loss": 1.0138177871704102,
+      "mean_token_accuracy": 0.6940690726041794,
+      "num_tokens": 17965115.0,
+      "step": 186
+    },
+    {
+      "entropy": 1.0318024158477783,
+      "epoch": 1.8038740920096852,
+      "grad_norm": 0.17989851534366608,
+      "learning_rate": 1.9215480198991466e-05,
+      "loss": 1.0178849697113037,
+      "mean_token_accuracy": 0.6889730393886566,
+      "num_tokens": 18062515.0,
+      "step": 187
+    },
+    {
+      "entropy": 1.0370707884430885,
+      "epoch": 1.8135593220338984,
+      "grad_norm": 0.18175791203975677,
+      "learning_rate": 1.895767401301994e-05,
+      "loss": 1.0233769416809082,
+      "mean_token_accuracy": 0.689442440867424,
+      "num_tokens": 18159901.0,
+      "step": 188
+    },
+    {
+      "entropy": 1.0387394651770592,
+      "epoch": 1.8232445520581115,
+      "grad_norm": 0.1763962209224701,
+      "learning_rate": 1.8700548465371874e-05,
+      "loss": 1.0317208766937256,
+      "mean_token_accuracy": 0.6874615550041199,
+      "num_tokens": 18257746.0,
+      "step": 189
+    },
+    {
+      "entropy": 1.016056202352047,
+      "epoch": 1.8329297820823245,
+      "grad_norm": 0.1811491698026657,
+      "learning_rate": 1.84441325199764e-05,
+      "loss": 1.0119330883026123,
+      "mean_token_accuracy": 0.6953575611114502,
+      "num_tokens": 18355948.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.9938200861215591,
+      "epoch": 1.8426150121065374,
+      "grad_norm": 0.1725209653377533,
+      "learning_rate": 1.818845506082943e-05,
+      "loss": 0.9812361598014832,
+      "mean_token_accuracy": 0.7028805017471313,
+      "num_tokens": 18453527.0,
+      "step": 191
+    },
+    {
+      "entropy": 1.031374305486679,
+      "epoch": 1.8523002421307506,
+      "grad_norm": 0.18246252834796906,
+      "learning_rate": 1.793354488874006e-05,
+      "loss": 1.0203440189361572,
+      "mean_token_accuracy": 0.6907096654176712,
+      "num_tokens": 18549884.0,
+      "step": 192
+    },
+    {
+      "entropy": 0.9885724261403084,
+      "epoch": 1.8619854721549638,
+      "grad_norm": 0.1804785132408142,
+      "learning_rate": 1.7679430718086243e-05,
+      "loss": 0.9796859622001648,
+      "mean_token_accuracy": 0.702710434794426,
+      "num_tokens": 18647122.0,
+      "step": 193
+    },
+    {
+      "entropy": 0.9969891235232353,
+      "epoch": 1.8716707021791767,
+      "grad_norm": 0.2115977555513382,
+      "learning_rate": 1.742614117358029e-05,
+      "loss": 0.9887296557426453,
+      "mean_token_accuracy": 0.6984992548823357,
+      "num_tokens": 18743129.0,
+      "step": 194
+    },
+    {
+      "entropy": 0.9905409216880798,
+      "epoch": 1.8813559322033897,
+      "grad_norm": 0.18363232910633087,
+      "learning_rate": 1.7173704787044446e-05,
+      "loss": 0.9921536445617676,
+      "mean_token_accuracy": 0.697743609547615,
+      "num_tokens": 18841235.0,
+      "step": 195
+    },
+    {
+      "entropy": 0.9967489168047905,
+      "epoch": 1.8910411622276029,
+      "grad_norm": 0.1787017434835434,
+      "learning_rate": 1.6922149994196878e-05,
+      "loss": 1.0015537738800049,
+      "mean_token_accuracy": 0.6942943632602692,
+      "num_tokens": 18938405.0,
+      "step": 196
+    },
+    {
+      "entropy": 1.017027884721756,
+      "epoch": 1.900726392251816,
+      "grad_norm": 0.1760513335466385,
+      "learning_rate": 1.667150513144856e-05,
+      "loss": 1.0226783752441406,
+      "mean_token_accuracy": 0.691545270383358,
+      "num_tokens": 19036103.0,
+      "step": 197
+    },
+    {
+      "entropy": 1.0095921084284782,
+      "epoch": 1.910411622276029,
+      "grad_norm": 0.1807909458875656,
+      "learning_rate": 1.6421798432711345e-05,
+      "loss": 1.004764437675476,
+      "mean_token_accuracy": 0.6964360028505325,
+      "num_tokens": 19131869.0,
+      "step": 198
+    },
+    {
+      "entropy": 1.0117909163236618,
+      "epoch": 1.9200968523002422,
+      "grad_norm": 0.18069379031658173,
+      "learning_rate": 1.617305802621748e-05,
+      "loss": 0.9964424967765808,
+      "mean_token_accuracy": 0.6977855116128922,
+      "num_tokens": 19229332.0,
+      "step": 199
+    },
+    {
+      "entropy": 1.0152455270290375,
+      "epoch": 1.9297820823244551,
+      "grad_norm": 0.1750374287366867,
+      "learning_rate": 1.5925311931351172e-05,
+      "loss": 1.0085126161575317,
+      "mean_token_accuracy": 0.6957272365689278,
+      "num_tokens": 19326511.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.0058443173766136,
+      "epoch": 1.9394673123486683,
+      "grad_norm": 0.17271851003170013,
+      "learning_rate": 1.567858805549229e-05,
+      "loss": 0.9887601733207703,
+      "mean_token_accuracy": 0.6980008259415627,
+      "num_tokens": 19423560.0,
+      "step": 201
+    },
+    {
+      "entropy": 1.0228681191802025,
+      "epoch": 1.9491525423728815,
+      "grad_norm": 0.17884734272956848,
+      "learning_rate": 1.5432914190872757e-05,
+      "loss": 1.0131609439849854,
+      "mean_token_accuracy": 0.692680798470974,
+      "num_tokens": 19520842.0,
+      "step": 202
+    },
+    {
+      "entropy": 1.0089232549071312,
+      "epoch": 1.9588377723970944,
+      "grad_norm": 0.16985908150672913,
+      "learning_rate": 1.5188318011445906e-05,
+      "loss": 1.0013556480407715,
+      "mean_token_accuracy": 0.6970010101795197,
+      "num_tokens": 19618823.0,
+      "step": 203
+    },
+    {
+      "entropy": 1.0016749277710915,
+      "epoch": 1.9685230024213074,
+      "grad_norm": 0.1778869777917862,
+      "learning_rate": 1.4944827069769123e-05,
+      "loss": 0.9919571876525879,
+      "mean_token_accuracy": 0.7001852169632912,
+      "num_tokens": 19716713.0,
+      "step": 204
+    },
+    {
+      "entropy": 1.0164751634001732,
+      "epoch": 1.9782082324455206,
+      "grad_norm": 0.18251122534275055,
+      "learning_rate": 1.4702468793900188e-05,
+      "loss": 1.0077866315841675,
+      "mean_token_accuracy": 0.6937248483300209,
+      "num_tokens": 19813324.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.0116839110851288,
+      "epoch": 1.9878934624697338,
+      "grad_norm": 0.17308309674263,
+      "learning_rate": 1.4461270484307643e-05,
+      "loss": 1.0083445310592651,
+      "mean_token_accuracy": 0.6930299252271652,
+      "num_tokens": 19910449.0,
+      "step": 206
+    },
+    {
+      "entropy": 1.0046918392181396,
+      "epoch": 1.9975786924939467,
+      "grad_norm": 0.17221423983573914,
+      "learning_rate": 1.4221259310795543e-05,
+      "loss": 0.9961509704589844,
+      "mean_token_accuracy": 0.6965227276086807,
+      "num_tokens": 20008513.0,
+      "step": 207
+    },
+    {
+      "entropy": 1.0230849385261536,
+      "epoch": 2.0,
+      "grad_norm": 0.35064876079559326,
+      "learning_rate": 1.3982462309442889e-05,
+      "loss": 1.0117878913879395,
+      "mean_token_accuracy": 0.6928033828735352,
+      "num_tokens": 20032830.0,
+      "step": 208
+    },
+    {
+      "entropy": 0.9964845478534698,
+      "epoch": 2.009685230024213,
+      "grad_norm": 0.17636848986148834,
+      "learning_rate": 1.3744906379558165e-05,
+      "loss": 0.9579638242721558,
+      "mean_token_accuracy": 0.705834411084652,
+      "num_tokens": 20129870.0,
+      "step": 209
+    },
+    {
+      "entropy": 1.0106598287820816,
+      "epoch": 2.0193704600484264,
+      "grad_norm": 0.1803821325302124,
+      "learning_rate": 1.3508618280649255e-05,
+      "loss": 0.9775141477584839,
+      "mean_token_accuracy": 0.7011803910136223,
+      "num_tokens": 20226728.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.0030332654714584,
+      "epoch": 2.029055690072639,
+      "grad_norm": 0.18877284228801727,
+      "learning_rate": 1.3273624629409082e-05,
+      "loss": 0.9775636196136475,
+      "mean_token_accuracy": 0.7004503831267357,
+      "num_tokens": 20321345.0,
+      "step": 211
+    },
+    {
+      "entropy": 1.0126233249902725,
+      "epoch": 2.0387409200968523,
+      "grad_norm": 0.18291965126991272,
+      "learning_rate": 1.3039951896717445e-05,
+      "loss": 0.9895316362380981,
+      "mean_token_accuracy": 0.6989523768424988,
+      "num_tokens": 20418998.0,
+      "step": 212
+    },
+    {
+      "entropy": 0.994785726070404,
+      "epoch": 2.0484261501210654,
+      "grad_norm": 0.17805488407611847,
+      "learning_rate": 1.2807626404659142e-05,
+      "loss": 0.9779917597770691,
+      "mean_token_accuracy": 0.7008749172091484,
+      "num_tokens": 20516839.0,
+      "step": 213
+    },
+    {
+      "entropy": 0.981712318956852,
+      "epoch": 2.0581113801452786,
+      "grad_norm": 0.1821916103363037,
+      "learning_rate": 1.2576674323558929e-05,
+      "loss": 0.9739880561828613,
+      "mean_token_accuracy": 0.7004686295986176,
+      "num_tokens": 20614245.0,
+      "step": 214
+    },
+    {
+      "entropy": 0.9724540486931801,
+      "epoch": 2.0677966101694913,
+      "grad_norm": 0.18674227595329285,
+      "learning_rate": 1.234712166903359e-05,
+      "loss": 0.9675310850143433,
+      "mean_token_accuracy": 0.7027364075183868,
+      "num_tokens": 20711954.0,
+      "step": 215
+    },
+    {
+      "entropy": 0.9729701578617096,
+      "epoch": 2.0774818401937045,
+      "grad_norm": 0.18655931949615479,
+      "learning_rate": 1.2118994299061376e-05,
+      "loss": 0.9668871760368347,
+      "mean_token_accuracy": 0.7026202082633972,
+      "num_tokens": 20807522.0,
+      "step": 216
+    },
+    {
+      "entropy": 0.9427368491888046,
+      "epoch": 2.0871670702179177,
+      "grad_norm": 0.17658740282058716,
+      "learning_rate": 1.1892317911069212e-05,
+      "loss": 0.9401254653930664,
+      "mean_token_accuracy": 0.709912471473217,
+      "num_tokens": 20905595.0,
+      "step": 217
+    },
+    {
+      "entropy": 0.9661683440208435,
+      "epoch": 2.096852300242131,
+      "grad_norm": 0.18123148381710052,
+      "learning_rate": 1.1667118039038064e-05,
+      "loss": 0.9703402519226074,
+      "mean_token_accuracy": 0.7052091136574745,
+      "num_tokens": 21002063.0,
+      "step": 218
+    },
+    {
+      "entropy": 0.9760492444038391,
+      "epoch": 2.106537530266344,
+      "grad_norm": 0.18296928703784943,
+      "learning_rate": 1.1443420050626625e-05,
+      "loss": 0.9792553782463074,
+      "mean_token_accuracy": 0.6997263431549072,
+      "num_tokens": 21099209.0,
+      "step": 219
+    },
+    {
+      "entropy": 0.9762522503733635,
+      "epoch": 2.116222760290557,
+      "grad_norm": 0.18274007737636566,
+      "learning_rate": 1.1221249144313778e-05,
+      "loss": 0.9795980453491211,
+      "mean_token_accuracy": 0.7015334367752075,
+      "num_tokens": 21196971.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.9688016548752785,
+      "epoch": 2.12590799031477,
+      "grad_norm": 0.1775023192167282,
+      "learning_rate": 1.1000630346560117e-05,
+      "loss": 0.9678329229354858,
+      "mean_token_accuracy": 0.7037922069430351,
+      "num_tokens": 21294641.0,
+      "step": 221
+    },
+    {
+      "entropy": 0.9825037941336632,
+      "epoch": 2.135593220338983,
+      "grad_norm": 0.1812477856874466,
+      "learning_rate": 1.078158850898883e-05,
+      "loss": 0.9817320108413696,
+      "mean_token_accuracy": 0.699578233063221,
+      "num_tokens": 21391604.0,
+      "step": 222
+    },
+    {
+      "entropy": 0.9769124090671539,
+      "epoch": 2.1452784503631963,
+      "grad_norm": 0.1791737824678421,
+      "learning_rate": 1.0564148305586296e-05,
+      "loss": 0.9674513339996338,
+      "mean_token_accuracy": 0.7023534849286079,
+      "num_tokens": 21488724.0,
+      "step": 223
+    },
+    {
+      "entropy": 1.0039660334587097,
+      "epoch": 2.154963680387409,
+      "grad_norm": 0.1795329749584198,
+      "learning_rate": 1.0348334229922677e-05,
+      "loss": 0.9883779287338257,
+      "mean_token_accuracy": 0.697204701602459,
+      "num_tokens": 21585010.0,
+      "step": 224
+    },
+    {
+      "entropy": 0.974880613386631,
+      "epoch": 2.164648910411622,
+      "grad_norm": 0.18022607266902924,
+      "learning_rate": 1.0134170592392836e-05,
+      "loss": 0.9638246297836304,
+      "mean_token_accuracy": 0.7034358531236649,
+      "num_tokens": 21682815.0,
+      "step": 225
+    },
+    {
+      "entropy": 0.9704831093549728,
+      "epoch": 2.1743341404358354,
+      "grad_norm": 0.17907658219337463,
+      "learning_rate": 9.9216815174779e-06,
+      "loss": 0.9586762189865112,
+      "mean_token_accuracy": 0.7059309855103493,
+      "num_tokens": 21780813.0,
+      "step": 226
+    },
+    {
+      "entropy": 0.9787583351135254,
+      "epoch": 2.1840193704600486,
+      "grad_norm": 0.17604723572731018,
+      "learning_rate": 9.710890941027722e-06,
+      "loss": 0.9592881202697754,
+      "mean_token_accuracy": 0.7065609246492386,
+      "num_tokens": 21876431.0,
+      "step": 227
+    },
+    {
+      "entropy": 0.997703529894352,
+      "epoch": 2.1937046004842613,
+      "grad_norm": 0.1816512644290924,
+      "learning_rate": 9.501822607564678e-06,
+      "loss": 0.983452320098877,
+      "mean_token_accuracy": 0.7003907486796379,
+      "num_tokens": 21974059.0,
+      "step": 228
+    },
+    {
+      "entropy": 0.9781674444675446,
+      "epoch": 2.2033898305084745,
+      "grad_norm": 0.17929424345493317,
+      "learning_rate": 9.29450006760894e-06,
+      "loss": 0.9610666632652283,
+      "mean_token_accuracy": 0.704089567065239,
+      "num_tokens": 22071550.0,
+      "step": 229
+    },
+    {
+      "entropy": 0.9733332246541977,
+      "epoch": 2.2130750605326877,
+      "grad_norm": 0.17849525809288025,
+      "learning_rate": 9.088946675025605e-06,
+      "loss": 0.9660186171531677,
+      "mean_token_accuracy": 0.7035543248057365,
+      "num_tokens": 22169195.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.9842173457145691,
+      "epoch": 2.222760290556901,
+      "grad_norm": 0.19081687927246094,
+      "learning_rate": 8.88518558439402e-06,
+      "loss": 0.9738259315490723,
+      "mean_token_accuracy": 0.700490303337574,
+      "num_tokens": 22264238.0,
+      "step": 231
+    },
+    {
+      "entropy": 0.9491748958826065,
+      "epoch": 2.232445520581114,
+      "grad_norm": 0.17600634694099426,
+      "learning_rate": 8.683239748399539e-06,
+      "loss": 0.9415581226348877,
+      "mean_token_accuracy": 0.7125281170010567,
+      "num_tokens": 22361030.0,
+      "step": 232
+    },
+    {
+      "entropy": 0.9827496632933617,
+      "epoch": 2.2421307506053267,
+      "grad_norm": 0.18011201918125153,
+      "learning_rate": 8.483131915247968e-06,
+      "loss": 0.9686768054962158,
+      "mean_token_accuracy": 0.7024280056357384,
+      "num_tokens": 22458625.0,
+      "step": 233
+    },
+    {
+      "entropy": 0.9476987347006798,
+      "epoch": 2.25181598062954,
+      "grad_norm": 0.18301460146903992,
+      "learning_rate": 8.284884626103165e-06,
+      "loss": 0.9361698031425476,
+      "mean_token_accuracy": 0.7130703330039978,
+      "num_tokens": 22555860.0,
+      "step": 234
+    },
+    {
+      "entropy": 0.9788018614053726,
+      "epoch": 2.261501210653753,
+      "grad_norm": 0.2318762242794037,
+      "learning_rate": 8.088520212547831e-06,
+      "loss": 0.9733133316040039,
+      "mean_token_accuracy": 0.7013382613658905,
+      "num_tokens": 22653497.0,
+      "step": 235
+    },
+    {
+      "entropy": 0.9619165509939194,
+      "epoch": 2.2711864406779663,
+      "grad_norm": 0.17698800563812256,
+      "learning_rate": 7.894060794067987e-06,
+      "loss": 0.96214759349823,
+      "mean_token_accuracy": 0.7057057321071625,
+      "num_tokens": 22750035.0,
+      "step": 236
+    },
+    {
+      "entropy": 0.9869695082306862,
+      "epoch": 2.280871670702179,
+      "grad_norm": 0.2705877721309662,
+      "learning_rate": 7.701528275561348e-06,
+      "loss": 0.9887737035751343,
+      "mean_token_accuracy": 0.6977095082402229,
+      "num_tokens": 22846771.0,
+      "step": 237
+    },
+    {
+      "entropy": 0.9492938369512558,
+      "epoch": 2.290556900726392,
+      "grad_norm": 0.18406817317008972,
+      "learning_rate": 7.510944344869774e-06,
+      "loss": 0.94698166847229,
+      "mean_token_accuracy": 0.7091076523065567,
+      "num_tokens": 22940363.0,
+      "step": 238
+    },
+    {
+      "entropy": 0.9718275740742683,
+      "epoch": 2.3002421307506054,
+      "grad_norm": 0.17744125425815582,
+      "learning_rate": 7.3223304703363135e-06,
+      "loss": 0.9602268934249878,
+      "mean_token_accuracy": 0.7047717645764351,
+      "num_tokens": 23037771.0,
+      "step": 239
+    },
+    {
+      "entropy": 0.997263103723526,
+      "epoch": 2.3099273607748185,
+      "grad_norm": 0.17902712523937225,
+      "learning_rate": 7.13570789838687e-06,
+      "loss": 0.9941853880882263,
+      "mean_token_accuracy": 0.6950124651193619,
+      "num_tokens": 23135752.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.9768236577510834,
+      "epoch": 2.3196125907990313,
+      "grad_norm": 0.1829724907875061,
+      "learning_rate": 6.951097651136889e-06,
+      "loss": 0.9629150629043579,
+      "mean_token_accuracy": 0.7058604136109352,
+      "num_tokens": 23231870.0,
+      "step": 241
+    },
+    {
+      "entropy": 0.963948056101799,
+      "epoch": 2.3292978208232444,
+      "grad_norm": 0.17480851709842682,
+      "learning_rate": 6.768520524023348e-06,
+      "loss": 0.9460391998291016,
+      "mean_token_accuracy": 0.7081798017024994,
+      "num_tokens": 23330052.0,
+      "step": 242
+    },
+    {
+      "entropy": 0.9842232093214989,
+      "epoch": 2.3389830508474576,
+      "grad_norm": 0.1877422034740448,
+      "learning_rate": 6.587997083462197e-06,
+      "loss": 0.9799100756645203,
+      "mean_token_accuracy": 0.701549656689167,
+      "num_tokens": 23427424.0,
+      "step": 243
+    },
+    {
+      "entropy": 0.9613497108221054,
+      "epoch": 2.348668280871671,
+      "grad_norm": 0.18304583430290222,
+      "learning_rate": 6.4095476645317346e-06,
+      "loss": 0.9574179649353027,
+      "mean_token_accuracy": 0.7060347571969032,
+      "num_tokens": 23524347.0,
+      "step": 244
+    },
+    {
+      "entropy": 0.9711159467697144,
+      "epoch": 2.358353510895884,
+      "grad_norm": 0.1803869903087616,
+      "learning_rate": 6.23319236868189e-06,
+      "loss": 0.965959906578064,
+      "mean_token_accuracy": 0.7031628713011742,
+      "num_tokens": 23621709.0,
+      "step": 245
+    },
+    {
+      "entropy": 0.9694511815905571,
+      "epoch": 2.3680387409200967,
+      "grad_norm": 0.177644744515419,
+      "learning_rate": 6.058951061469931e-06,
+      "loss": 0.9606213569641113,
+      "mean_token_accuracy": 0.7050533071160316,
+      "num_tokens": 23718487.0,
+      "step": 246
+    },
+    {
+      "entropy": 0.9582285434007645,
+      "epoch": 2.37772397094431,
+      "grad_norm": 0.1824142336845398,
+      "learning_rate": 5.886843370322692e-06,
+      "loss": 0.9482771158218384,
+      "mean_token_accuracy": 0.7099712938070297,
+      "num_tokens": 23815121.0,
+      "step": 247
+    },
+    {
+      "entropy": 0.9778466373682022,
+      "epoch": 2.387409200968523,
+      "grad_norm": 0.18152157962322235,
+      "learning_rate": 5.7168886823256355e-06,
+      "loss": 0.9651317596435547,
+      "mean_token_accuracy": 0.7042052671313286,
+      "num_tokens": 23912149.0,
+      "step": 248
+    },
+    {
+      "entropy": 0.9690630733966827,
+      "epoch": 2.3970944309927362,
+      "grad_norm": 0.2793174386024475,
+      "learning_rate": 5.549106142039018e-06,
+      "loss": 0.9617438316345215,
+      "mean_token_accuracy": 0.7056540101766586,
+      "num_tokens": 24009273.0,
+      "step": 249
+    },
+    {
+      "entropy": 0.9750760719180107,
+      "epoch": 2.406779661016949,
+      "grad_norm": 0.17681074142456055,
+      "learning_rate": 5.38351464934132e-06,
+      "loss": 0.9575906991958618,
+      "mean_token_accuracy": 0.7047781273722649,
+      "num_tokens": 24107354.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.9507534727454185,
+      "epoch": 2.416464891041162,
+      "grad_norm": 0.18025094270706177,
+      "learning_rate": 5.220132857300286e-06,
+      "loss": 0.9377824664115906,
+      "mean_token_accuracy": 0.7107547968626022,
+      "num_tokens": 24204737.0,
+      "step": 251
+    },
+    {
+      "entropy": 0.9844745770096779,
+      "epoch": 2.4261501210653753,
+      "grad_norm": 0.17789708077907562,
+      "learning_rate": 5.058979170071754e-06,
+      "loss": 0.9699742197990417,
+      "mean_token_accuracy": 0.7021560966968536,
+      "num_tokens": 24301999.0,
+      "step": 252
+    },
+    {
+      "entropy": 0.9685339778661728,
+      "epoch": 2.4358353510895885,
+      "grad_norm": 0.17267119884490967,
+      "learning_rate": 4.900071740826489e-06,
+      "loss": 0.9629323482513428,
+      "mean_token_accuracy": 0.7057926878333092,
+      "num_tokens": 24398022.0,
+      "step": 253
+    },
+    {
+      "entropy": 0.965476855635643,
+      "epoch": 2.4455205811138017,
+      "grad_norm": 0.18101957440376282,
+      "learning_rate": 4.743428469705335e-06,
+      "loss": 0.955784022808075,
+      "mean_token_accuracy": 0.7057157456874847,
+      "num_tokens": 24494712.0,
+      "step": 254
+    },
+    {
+      "entropy": 0.9631841853260994,
+      "epoch": 2.4552058111380144,
+      "grad_norm": 0.17928491532802582,
+      "learning_rate": 4.58906700180286e-06,
+      "loss": 0.9480952024459839,
+      "mean_token_accuracy": 0.7076999545097351,
+      "num_tokens": 24591559.0,
+      "step": 255
+    },
+    {
+      "entropy": 0.9611305594444275,
+      "epoch": 2.4648910411622276,
+      "grad_norm": 0.17541737854480743,
+      "learning_rate": 4.437004725179713e-06,
+      "loss": 0.9549427032470703,
+      "mean_token_accuracy": 0.7060576677322388,
+      "num_tokens": 24689494.0,
+      "step": 256
+    },
+    {
+      "entropy": 0.9543889611959457,
+      "epoch": 2.4745762711864407,
+      "grad_norm": 0.1807403564453125,
+      "learning_rate": 4.2872587689039484e-06,
+      "loss": 0.9523176550865173,
+      "mean_token_accuracy": 0.7074451968073845,
+      "num_tokens": 24786283.0,
+      "step": 257
+    },
+    {
+      "entropy": 0.9686412364244461,
+      "epoch": 2.484261501210654,
+      "grad_norm": 0.19152818620204926,
+      "learning_rate": 4.139846001121514e-06,
+      "loss": 0.9642741680145264,
+      "mean_token_accuracy": 0.7037565931677818,
+      "num_tokens": 24883368.0,
+      "step": 258
+    },
+    {
+      "entropy": 0.9706502631306648,
+      "epoch": 2.4939467312348667,
+      "grad_norm": 0.18040058016777039,
+      "learning_rate": 3.9947830271561435e-06,
+      "loss": 0.9643933773040771,
+      "mean_token_accuracy": 0.7022472470998764,
+      "num_tokens": 24979284.0,
+      "step": 259
+    },
+    {
+      "entropy": 0.9656544327735901,
+      "epoch": 2.50363196125908,
+      "grad_norm": 0.17672894895076752,
+      "learning_rate": 3.852086187638846e-06,
+      "loss": 0.9573478102684021,
+      "mean_token_accuracy": 0.7059909179806709,
+      "num_tokens": 25076441.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.9618352204561234,
+      "epoch": 2.513317191283293,
+      "grad_norm": 0.1795944720506668,
+      "learning_rate": 3.711771556667218e-06,
+      "loss": 0.9508000612258911,
+      "mean_token_accuracy": 0.7096266373991966,
+      "num_tokens": 25174061.0,
+      "step": 261
+    },
+    {
+      "entropy": 0.9842494279146194,
+      "epoch": 2.523002421307506,
+      "grad_norm": 0.17950080335140228,
+      "learning_rate": 3.57385493999477e-06,
+      "loss": 0.9755753874778748,
+      "mean_token_accuracy": 0.7001539990305901,
+      "num_tokens": 25272103.0,
+      "step": 262
+    },
+    {
+      "entropy": 0.9461641684174538,
+      "epoch": 2.5326876513317194,
+      "grad_norm": 0.18211127817630768,
+      "learning_rate": 3.438351873250492e-06,
+      "loss": 0.9435604810714722,
+      "mean_token_accuracy": 0.7120839208364487,
+      "num_tokens": 25368715.0,
+      "step": 263
+    },
+    {
+      "entropy": 0.9493983089923859,
+      "epoch": 2.542372881355932,
+      "grad_norm": 0.17652387917041779,
+      "learning_rate": 3.305277620188826e-06,
+      "loss": 0.9497734904289246,
+      "mean_token_accuracy": 0.7066553607583046,
+      "num_tokens": 25466014.0,
+      "step": 264
+    },
+    {
+      "entropy": 0.9618336409330368,
+      "epoch": 2.5520581113801453,
+      "grad_norm": 0.2031353861093521,
+      "learning_rate": 3.1746471709702964e-06,
+      "loss": 0.9585392475128174,
+      "mean_token_accuracy": 0.7073178961873055,
+      "num_tokens": 25563415.0,
+      "step": 265
+    },
+    {
+      "entropy": 0.979316346347332,
+      "epoch": 2.5617433414043584,
+      "grad_norm": 0.18086563050746918,
+      "learning_rate": 3.0464752404729484e-06,
+      "loss": 0.975259006023407,
+      "mean_token_accuracy": 0.6986003741621971,
+      "num_tokens": 25661061.0,
+      "step": 266
+    },
+    {
+      "entropy": 0.9475531205534935,
+      "epoch": 2.571428571428571,
+      "grad_norm": 0.1784556657075882,
+      "learning_rate": 2.920776266634767e-06,
+      "loss": 0.9324929118156433,
+      "mean_token_accuracy": 0.7109888419508934,
+      "num_tokens": 25757833.0,
+      "step": 267
+    },
+    {
+      "entropy": 0.9776372313499451,
+      "epoch": 2.5811138014527844,
+      "grad_norm": 0.18078093230724335,
+      "learning_rate": 2.797564408827319e-06,
+      "loss": 0.963403582572937,
+      "mean_token_accuracy": 0.7044354677200317,
+      "num_tokens": 25853862.0,
+      "step": 268
+    },
+    {
+      "entropy": 0.9847007617354393,
+      "epoch": 2.5907990314769975,
+      "grad_norm": 0.17774543166160583,
+      "learning_rate": 2.676853546260791e-06,
+      "loss": 0.9728066921234131,
+      "mean_token_accuracy": 0.7033195123076439,
+      "num_tokens": 25951398.0,
+      "step": 269
+    },
+    {
+      "entropy": 0.9523753523826599,
+      "epoch": 2.6004842615012107,
+      "grad_norm": 0.1763666272163391,
+      "learning_rate": 2.558657276420526e-06,
+      "loss": 0.94881671667099,
+      "mean_token_accuracy": 0.7090617269277573,
+      "num_tokens": 26049292.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.9992275238037109,
+      "epoch": 2.610169491525424,
+      "grad_norm": 0.1775253713130951,
+      "learning_rate": 2.442988913535374e-06,
+      "loss": 0.9929525852203369,
+      "mean_token_accuracy": 0.6952389478683472,
+      "num_tokens": 26146570.0,
+      "step": 271
+    },
+    {
+      "entropy": 0.9768842831254005,
+      "epoch": 2.619854721549637,
+      "grad_norm": 0.18197324872016907,
+      "learning_rate": 2.3298614870778835e-06,
+      "loss": 0.9748570322990417,
+      "mean_token_accuracy": 0.7011555060744286,
+      "num_tokens": 26244193.0,
+      "step": 272
+    },
+    {
+      "entropy": 0.9624940007925034,
+      "epoch": 2.62953995157385,
+      "grad_norm": 0.1776597946882248,
+      "learning_rate": 2.219287740296605e-06,
+      "loss": 0.9518307447433472,
+      "mean_token_accuracy": 0.7069382146000862,
+      "num_tokens": 26341731.0,
+      "step": 273
+    },
+    {
+      "entropy": 0.9795135259628296,
+      "epoch": 2.639225181598063,
+      "grad_norm": 0.20650646090507507,
+      "learning_rate": 2.1112801287806378e-06,
+      "loss": 0.9681733846664429,
+      "mean_token_accuracy": 0.7024837136268616,
+      "num_tokens": 26438264.0,
+      "step": 274
+    },
+    {
+      "entropy": 0.966120146214962,
+      "epoch": 2.648910411622276,
+      "grad_norm": 0.18002614378929138,
+      "learning_rate": 2.0058508190565315e-06,
+      "loss": 0.9650594592094421,
+      "mean_token_accuracy": 0.7058759182691574,
+      "num_tokens": 26536362.0,
+      "step": 275
+    },
+    {
+      "entropy": 0.9701863154768944,
+      "epoch": 2.658595641646489,
+      "grad_norm": 0.1820772886276245,
+      "learning_rate": 1.9030116872178316e-06,
+      "loss": 0.9537088871002197,
+      "mean_token_accuracy": 0.7048317566514015,
+      "num_tokens": 26633878.0,
+      "step": 276
+    },
+    {
+      "entropy": 0.9413173869252205,
+      "epoch": 2.668280871670702,
+      "grad_norm": 0.1845010668039322,
+      "learning_rate": 1.8027743175872664e-06,
+      "loss": 0.930335283279419,
+      "mean_token_accuracy": 0.7123815342783928,
+      "num_tokens": 26731252.0,
+      "step": 277
+    },
+    {
+      "entropy": 0.9709818363189697,
+      "epoch": 2.6779661016949152,
+      "grad_norm": 0.18073733150959015,
+      "learning_rate": 1.7051500014118454e-06,
+      "loss": 0.9600517153739929,
+      "mean_token_accuracy": 0.7052488699555397,
+      "num_tokens": 26827337.0,
+      "step": 278
+    },
+    {
+      "entropy": 0.9786422699689865,
+      "epoch": 2.6876513317191284,
+      "grad_norm": 0.17675380408763885,
+      "learning_rate": 1.610149735590949e-06,
+      "loss": 0.9669743776321411,
+      "mean_token_accuracy": 0.7031452357769012,
+      "num_tokens": 26924020.0,
+      "step": 279
+    },
+    {
+      "entropy": 0.9676607623696327,
+      "epoch": 2.6973365617433416,
+      "grad_norm": 0.1773083359003067,
+      "learning_rate": 1.517784221437568e-06,
+      "loss": 0.9580140113830566,
+      "mean_token_accuracy": 0.7057247683405876,
+      "num_tokens": 27020699.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.9538565576076508,
+      "epoch": 2.7070217917675548,
+      "grad_norm": 0.17378197610378265,
+      "learning_rate": 1.428063863472895e-06,
+      "loss": 0.9394268989562988,
+      "mean_token_accuracy": 0.7104142755270004,
+      "num_tokens": 27117563.0,
+      "step": 281
+    },
+    {
+      "entropy": 0.9721490517258644,
+      "epoch": 2.7167070217917675,
+      "grad_norm": 0.1738930642604828,
+      "learning_rate": 1.3409987682542619e-06,
+      "loss": 0.9649605751037598,
+      "mean_token_accuracy": 0.70272546261549,
+      "num_tokens": 27215426.0,
+      "step": 282
+    },
+    {
+      "entropy": 0.9914449229836464,
+      "epoch": 2.7263922518159807,
+      "grad_norm": 0.18021667003631592,
+      "learning_rate": 1.2565987432367033e-06,
+      "loss": 0.9891276359558105,
+      "mean_token_accuracy": 0.6978868618607521,
+      "num_tokens": 27312182.0,
+      "step": 283
+    },
+    {
+      "entropy": 0.9563105776906013,
+      "epoch": 2.736077481840194,
+      "grad_norm": 0.17998550832271576,
+      "learning_rate": 1.1748732956682025e-06,
+      "loss": 0.9561561346054077,
+      "mean_token_accuracy": 0.7088183015584946,
+      "num_tokens": 27409659.0,
+      "step": 284
+    },
+    {
+      "entropy": 0.9712988957762718,
+      "epoch": 2.7457627118644066,
+      "grad_norm": 0.17391616106033325,
+      "learning_rate": 1.095831631518729e-06,
+      "loss": 0.9587939977645874,
+      "mean_token_accuracy": 0.7066119387745857,
+      "num_tokens": 27507879.0,
+      "step": 285
+    },
+    {
+      "entropy": 0.9707677364349365,
+      "epoch": 2.7554479418886197,
+      "grad_norm": 0.17616266012191772,
+      "learning_rate": 1.0194826544432518e-06,
+      "loss": 0.9611558318138123,
+      "mean_token_accuracy": 0.7038435861468315,
+      "num_tokens": 27604015.0,
+      "step": 286
+    },
+    {
+      "entropy": 0.9944893270730972,
+      "epoch": 2.765133171912833,
+      "grad_norm": 0.17953266203403473,
+      "learning_rate": 9.458349647787662e-07,
+      "loss": 0.9858497977256775,
+      "mean_token_accuracy": 0.7003906443715096,
+      "num_tokens": 27701604.0,
+      "step": 287
+    },
+    {
+      "entropy": 0.9667356237769127,
+      "epoch": 2.774818401937046,
+      "grad_norm": 0.1791837066411972,
+      "learning_rate": 8.748968585755274e-07,
+      "loss": 0.9597692489624023,
+      "mean_token_accuracy": 0.7067226842045784,
+      "num_tokens": 27799036.0,
+      "step": 288
+    },
+    {
+      "entropy": 0.9621480256319046,
+      "epoch": 2.7845036319612593,
+      "grad_norm": 0.17390349507331848,
+      "learning_rate": 8.066763266625282e-07,
+      "loss": 0.9528490900993347,
+      "mean_token_accuracy": 0.7082757875323296,
+      "num_tokens": 27896645.0,
+      "step": 289
+    },
+    {
+      "entropy": 0.9826484322547913,
+      "epoch": 2.794188861985472,
+      "grad_norm": 0.1810206174850464,
+      "learning_rate": 7.411810537473752e-07,
+      "loss": 0.978987455368042,
+      "mean_token_accuracy": 0.7006519734859467,
+      "num_tokens": 27992284.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.9421405792236328,
+      "epoch": 2.803874092009685,
+      "grad_norm": 0.1830003559589386,
+      "learning_rate": 6.784184175506358e-07,
+      "loss": 0.9335393905639648,
+      "mean_token_accuracy": 0.7116543427109718,
+      "num_tokens": 28086308.0,
+      "step": 291
+    },
+    {
+      "entropy": 0.9940215274691582,
+      "epoch": 2.8135593220338984,
+      "grad_norm": 0.18031619489192963,
+      "learning_rate": 6.18395487974796e-07,
+      "loss": 0.9852080345153809,
+      "mean_token_accuracy": 0.6984784379601479,
+      "num_tokens": 28181742.0,
+      "step": 292
+    },
+    {
+      "entropy": 0.9698458909988403,
+      "epoch": 2.8232445520581115,
+      "grad_norm": 0.2049950659275055,
+      "learning_rate": 5.611190263078464e-07,
+      "loss": 0.9612412452697754,
+      "mean_token_accuracy": 0.7049182057380676,
+      "num_tokens": 28278865.0,
+      "step": 293
+    },
+    {
+      "entropy": 0.9680915996432304,
+      "epoch": 2.8329297820823243,
+      "grad_norm": 0.17655394971370697,
+      "learning_rate": 5.065954844616721e-07,
+      "loss": 0.954717755317688,
+      "mean_token_accuracy": 0.7056088149547577,
+      "num_tokens": 28376330.0,
+      "step": 294
+    },
+    {
+      "entropy": 0.9611566066741943,
+      "epoch": 2.8426150121065374,
+      "grad_norm": 0.18384459614753723,
+      "learning_rate": 4.548310042452736e-07,
+      "loss": 0.9458622336387634,
+      "mean_token_accuracy": 0.7086897045373917,
+      "num_tokens": 28473794.0,
+      "step": 295
+    },
+    {
+      "entropy": 0.9896327629685402,
+      "epoch": 2.8523002421307506,
+      "grad_norm": 0.17493842542171478,
+      "learning_rate": 4.05831416672911e-07,
+      "loss": 0.981327474117279,
+      "mean_token_accuracy": 0.7018685564398766,
+      "num_tokens": 28571821.0,
+      "step": 296
+    },
+    {
+      "entropy": 0.9739344343543053,
+      "epoch": 2.861985472154964,
+      "grad_norm": 0.17228510975837708,
+      "learning_rate": 3.5960224130728857e-07,
+      "loss": 0.966514527797699,
+      "mean_token_accuracy": 0.7024144753813744,
+      "num_tokens": 28669376.0,
+      "step": 297
+    },
+    {
+      "entropy": 0.9500202685594559,
+      "epoch": 2.871670702179177,
+      "grad_norm": 0.18412764370441437,
+      "learning_rate": 3.1614868563778487e-07,
+      "loss": 0.9380296468734741,
+      "mean_token_accuracy": 0.7122134119272232,
+      "num_tokens": 28766843.0,
+      "step": 298
+    },
+    {
+      "entropy": 0.9886195659637451,
+      "epoch": 2.8813559322033897,
+      "grad_norm": 0.174481600522995,
+      "learning_rate": 2.7547564449386664e-07,
+      "loss": 0.9791684150695801,
+      "mean_token_accuracy": 0.7002565562725067,
+      "num_tokens": 28863956.0,
+      "step": 299
+    },
+    {
+      "entropy": 0.9633003771305084,
+      "epoch": 2.891041162227603,
+      "grad_norm": 0.17495881021022797,
+      "learning_rate": 2.3758769949370762e-07,
+      "loss": 0.9534497261047363,
+      "mean_token_accuracy": 0.7069478556513786,
+      "num_tokens": 28961301.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.9598926454782486,
+      "epoch": 2.900726392251816,
+      "grad_norm": 0.18199527263641357,
+      "learning_rate": 2.0248911852807916e-07,
+      "loss": 0.9594986438751221,
+      "mean_token_accuracy": 0.7057793438434601,
+      "num_tokens": 29058457.0,
+      "step": 301
+    },
+    {
+      "entropy": 0.9787684008479118,
+      "epoch": 2.910411622276029,
+      "grad_norm": 0.17845535278320312,
+      "learning_rate": 1.7018385527961533e-07,
+      "loss": 0.9728447794914246,
+      "mean_token_accuracy": 0.7033894285559654,
+      "num_tokens": 29155076.0,
+      "step": 302
+    },
+    {
+      "entropy": 0.9754829704761505,
+      "epoch": 2.920096852300242,
+      "grad_norm": 0.17794829607009888,
+      "learning_rate": 1.406755487774386e-07,
+      "loss": 0.964917778968811,
+      "mean_token_accuracy": 0.7027197331190109,
+      "num_tokens": 29251719.0,
+      "step": 303
+    },
+    {
+      "entropy": 0.9619567468762398,
+      "epoch": 2.929782082324455,
+      "grad_norm": 0.1754407286643982,
+      "learning_rate": 1.1396752298723501e-07,
+      "loss": 0.9576239585876465,
+      "mean_token_accuracy": 0.706841915845871,
+      "num_tokens": 29347947.0,
+      "step": 304
+    },
+    {
+      "entropy": 0.9863602966070175,
+      "epoch": 2.9394673123486683,
+      "grad_norm": 0.17757053673267365,
+      "learning_rate": 9.006278643683696e-08,
+      "loss": 0.9789604544639587,
+      "mean_token_accuracy": 0.7016789391636848,
+      "num_tokens": 29445507.0,
+      "step": 305
+    },
+    {
+      "entropy": 0.9918009713292122,
+      "epoch": 2.9491525423728815,
+      "grad_norm": 0.18638379871845245,
+      "learning_rate": 6.896403187732215e-08,
+      "loss": 0.9897012710571289,
+      "mean_token_accuracy": 0.6980951949954033,
+      "num_tokens": 29542632.0,
+      "step": 306
+    },
+    {
+      "entropy": 0.9726842790842056,
+      "epoch": 2.9588377723970947,
+      "grad_norm": 0.17771297693252563,
+      "learning_rate": 5.0673635979686665e-08,
+      "loss": 0.9664013385772705,
+      "mean_token_accuracy": 0.7052607387304306,
+      "num_tokens": 29639414.0,
+      "step": 307
+    },
+    {
+      "entropy": 0.9628136456012726,
+      "epoch": 2.9685230024213074,
+      "grad_norm": 0.1774171143770218,
+      "learning_rate": 3.5193659067131455e-08,
+      "loss": 0.95008385181427,
+      "mean_token_accuracy": 0.7064594179391861,
+      "num_tokens": 29734105.0,
+      "step": 308
+    },
+    {
+      "entropy": 0.9817704111337662,
+      "epoch": 2.9782082324455206,
+      "grad_norm": 0.17644129693508148,
+      "learning_rate": 2.2525844882964607e-08,
+      "loss": 0.9797781705856323,
+      "mean_token_accuracy": 0.6994470730423927,
+      "num_tokens": 29832091.0,
+      "step": 309
+    },
+    {
+      "entropy": 0.9816061109304428,
+      "epoch": 2.9878934624697338,
+      "grad_norm": 0.17975273728370667,
+      "learning_rate": 1.267162039418901e-08,
+      "loss": 0.976552426815033,
+      "mean_token_accuracy": 0.698183961212635,
+      "num_tokens": 29929348.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.961225301027298,
+      "epoch": 2.9975786924939465,
+      "grad_norm": 0.17494691908359528,
+      "learning_rate": 5.632095630764278e-09,
+      "loss": 0.9551951885223389,
+      "mean_token_accuracy": 0.7057754248380661,
+      "num_tokens": 30025466.0,
+      "step": 311
+    },
+    {
+      "entropy": 0.9253799617290497,
+      "epoch": 3.0,
+      "grad_norm": 0.3442399203777313,
+      "learning_rate": 1.4080635605512316e-09,
+      "loss": 0.9125053882598877,
+      "mean_token_accuracy": 0.716167688369751,
+      "num_tokens": 30049245.0,
+      "step": 312
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 312,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.2267649894294487e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}