Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-70B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-70B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-70B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "up_proj",
+    "q_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:860b753a0a96db8f213b61d12878c465439bed64d7f4e9f55590793aaba0adce
+size 3313653480

chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

trainer_state.json

@@ -0,0 +1,1074 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 104,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4476254880428314,
+      "epoch": 0.009685230024213076,
+      "grad_norm": 1.3588989973068237,
+      "learning_rate": 0.0,
+      "loss": 2.2350821495056152,
+      "mean_token_accuracy": 0.5031501688063145,
+      "num_tokens": 96574.0,
+      "step": 1
+    },
+    {
+      "entropy": 1.4007079899311066,
+      "epoch": 0.01937046004842615,
+      "grad_norm": 1.2849924564361572,
+      "learning_rate": 3.125e-06,
+      "loss": 2.14509654045105,
+      "mean_token_accuracy": 0.5205462127923965,
+      "num_tokens": 191520.0,
+      "step": 2
+    },
+    {
+      "entropy": 1.4708476215600967,
+      "epoch": 0.029055690072639227,
+      "grad_norm": 1.2314231395721436,
+      "learning_rate": 6.25e-06,
+      "loss": 2.158306121826172,
+      "mean_token_accuracy": 0.5159582570195198,
+      "num_tokens": 287631.0,
+      "step": 3
+    },
+    {
+      "entropy": 1.5651142746210098,
+      "epoch": 0.0387409200968523,
+      "grad_norm": 0.9785442352294922,
+      "learning_rate": 9.375000000000001e-06,
+      "loss": 2.0755105018615723,
+      "mean_token_accuracy": 0.5181002020835876,
+      "num_tokens": 384739.0,
+      "step": 4
+    },
+    {
+      "entropy": 1.579958975315094,
+      "epoch": 0.048426150121065374,
+      "grad_norm": 0.7276659607887268,
+      "learning_rate": 1.25e-05,
+      "loss": 1.930289626121521,
+      "mean_token_accuracy": 0.5408441498875618,
+      "num_tokens": 481674.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.8359138667583466,
+      "epoch": 0.05811138014527845,
+      "grad_norm": 0.5072433352470398,
+      "learning_rate": 1.5625e-05,
+      "loss": 1.9374589920043945,
+      "mean_token_accuracy": 0.5299954265356064,
+      "num_tokens": 576090.0,
+      "step": 6
+    },
+    {
+      "entropy": 1.8671298921108246,
+      "epoch": 0.06779661016949153,
+      "grad_norm": 0.44714638590812683,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 1.8663867712020874,
+      "mean_token_accuracy": 0.5436736792325974,
+      "num_tokens": 673951.0,
+      "step": 7
+    },
+    {
+      "entropy": 1.933446392416954,
+      "epoch": 0.0774818401937046,
+      "grad_norm": 0.5136867761611938,
+      "learning_rate": 2.1875e-05,
+      "loss": 1.7743638753890991,
+      "mean_token_accuracy": 0.5584963485598564,
+      "num_tokens": 771899.0,
+      "step": 8
+    },
+    {
+      "entropy": 1.9850385338068008,
+      "epoch": 0.08716707021791767,
+      "grad_norm": 0.5442814826965332,
+      "learning_rate": 2.5e-05,
+      "loss": 1.7758773565292358,
+      "mean_token_accuracy": 0.5573008134961128,
+      "num_tokens": 868206.0,
+      "step": 9
+    },
+    {
+      "entropy": 1.9424707889556885,
+      "epoch": 0.09685230024213075,
+      "grad_norm": 0.5084043741226196,
+      "learning_rate": 2.8125000000000003e-05,
+      "loss": 1.7394293546676636,
+      "mean_token_accuracy": 0.5608929991722107,
+      "num_tokens": 964219.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.8063177168369293,
+      "epoch": 0.10653753026634383,
+      "grad_norm": 0.3967217206954956,
+      "learning_rate": 3.125e-05,
+      "loss": 1.6807222366333008,
+      "mean_token_accuracy": 0.5679514929652214,
+      "num_tokens": 1062125.0,
+      "step": 11
+    },
+    {
+      "entropy": 1.638314574956894,
+      "epoch": 0.1162227602905569,
+      "grad_norm": 0.5107802152633667,
+      "learning_rate": 3.4375e-05,
+      "loss": 1.6123173236846924,
+      "mean_token_accuracy": 0.583977460861206,
+      "num_tokens": 1159319.0,
+      "step": 12
+    },
+    {
+      "entropy": 1.5576547533273697,
+      "epoch": 0.12590799031476999,
+      "grad_norm": 0.3253207802772522,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 1.6140292882919312,
+      "mean_token_accuracy": 0.5774664506316185,
+      "num_tokens": 1256881.0,
+      "step": 13
+    },
+    {
+      "entropy": 1.4602049440145493,
+      "epoch": 0.13559322033898305,
+      "grad_norm": 0.3317743241786957,
+      "learning_rate": 4.0625000000000005e-05,
+      "loss": 1.5576837062835693,
+      "mean_token_accuracy": 0.586665965616703,
+      "num_tokens": 1351620.0,
+      "step": 14
+    },
+    {
+      "entropy": 1.4226532131433487,
+      "epoch": 0.14527845036319612,
+      "grad_norm": 0.2991933524608612,
+      "learning_rate": 4.375e-05,
+      "loss": 1.5325368642807007,
+      "mean_token_accuracy": 0.5934261903166771,
+      "num_tokens": 1449026.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.453083336353302,
+      "epoch": 0.1549636803874092,
+      "grad_norm": 0.26260727643966675,
+      "learning_rate": 4.6875e-05,
+      "loss": 1.5117604732513428,
+      "mean_token_accuracy": 0.5941561385989189,
+      "num_tokens": 1546355.0,
+      "step": 16
+    },
+    {
+      "entropy": 1.4769755005836487,
+      "epoch": 0.16464891041162227,
+      "grad_norm": 0.24356529116630554,
+      "learning_rate": 5e-05,
+      "loss": 1.5000947713851929,
+      "mean_token_accuracy": 0.5963850915431976,
+      "num_tokens": 1644308.0,
+      "step": 17
+    },
+    {
+      "entropy": 1.4608993381261826,
+      "epoch": 0.17433414043583534,
+      "grad_norm": 0.27541783452033997,
+      "learning_rate": 4.999859193643945e-05,
+      "loss": 1.4296932220458984,
+      "mean_token_accuracy": 0.6135347038507462,
+      "num_tokens": 1741625.0,
+      "step": 18
+    },
+    {
+      "entropy": 1.4715198874473572,
+      "epoch": 0.18401937046004843,
+      "grad_norm": 0.24254612624645233,
+      "learning_rate": 4.999436790436924e-05,
+      "loss": 1.4109866619110107,
+      "mean_token_accuracy": 0.6162872388958931,
+      "num_tokens": 1839077.0,
+      "step": 19
+    },
+    {
+      "entropy": 1.4909484386444092,
+      "epoch": 0.1937046004842615,
+      "grad_norm": 0.24249622225761414,
+      "learning_rate": 4.9987328379605816e-05,
+      "loss": 1.4181320667266846,
+      "mean_token_accuracy": 0.6092976555228233,
+      "num_tokens": 1936754.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.468568578362465,
+      "epoch": 0.2033898305084746,
+      "grad_norm": 0.21892383694648743,
+      "learning_rate": 4.9977474155117045e-05,
+      "loss": 1.4207193851470947,
+      "mean_token_accuracy": 0.6143926084041595,
+      "num_tokens": 2034049.0,
+      "step": 21
+    },
+    {
+      "entropy": 1.4144536703824997,
+      "epoch": 0.21307506053268765,
+      "grad_norm": 0.24802649021148682,
+      "learning_rate": 4.9964806340932865e-05,
+      "loss": 1.3823838233947754,
+      "mean_token_accuracy": 0.6196955218911171,
+      "num_tokens": 2131193.0,
+      "step": 22
+    },
+    {
+      "entropy": 1.4027946889400482,
+      "epoch": 0.22276029055690072,
+      "grad_norm": 0.2114405333995819,
+      "learning_rate": 4.9949326364020314e-05,
+      "loss": 1.3788299560546875,
+      "mean_token_accuracy": 0.6205098181962967,
+      "num_tokens": 2228554.0,
+      "step": 23
+    },
+    {
+      "entropy": 1.3630587011575699,
+      "epoch": 0.2324455205811138,
+      "grad_norm": 0.18567043542861938,
+      "learning_rate": 4.993103596812268e-05,
+      "loss": 1.3441742658615112,
+      "mean_token_accuracy": 0.6255069524049759,
+      "num_tokens": 2325761.0,
+      "step": 24
+    },
+    {
+      "entropy": 1.3504581600427628,
+      "epoch": 0.24213075060532688,
+      "grad_norm": 0.18140752613544464,
+      "learning_rate": 4.9909937213563165e-05,
+      "loss": 1.3367418050765991,
+      "mean_token_accuracy": 0.6294924318790436,
+      "num_tokens": 2420230.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.3556479513645172,
+      "epoch": 0.25181598062953997,
+      "grad_norm": 0.19952309131622314,
+      "learning_rate": 4.988603247701277e-05,
+      "loss": 1.329459547996521,
+      "mean_token_accuracy": 0.6290309652686119,
+      "num_tokens": 2515756.0,
+      "step": 26
+    },
+    {
+      "entropy": 1.3174784928560257,
+      "epoch": 0.26150121065375304,
+      "grad_norm": 0.2040238082408905,
+      "learning_rate": 4.985932445122257e-05,
+      "loss": 1.3146334886550903,
+      "mean_token_accuracy": 0.6350805461406708,
+      "num_tokens": 2611239.0,
+      "step": 27
+    },
+    {
+      "entropy": 1.337596833705902,
+      "epoch": 0.2711864406779661,
+      "grad_norm": 0.1924348771572113,
+      "learning_rate": 4.982981614472039e-05,
+      "loss": 1.325057864189148,
+      "mean_token_accuracy": 0.6268098577857018,
+      "num_tokens": 2708463.0,
+      "step": 28
+    },
+    {
+      "entropy": 1.295740321278572,
+      "epoch": 0.28087167070217917,
+      "grad_norm": 0.18018221855163574,
+      "learning_rate": 4.979751088147192e-05,
+      "loss": 1.2943141460418701,
+      "mean_token_accuracy": 0.6371493935585022,
+      "num_tokens": 2806105.0,
+      "step": 29
+    },
+    {
+      "entropy": 1.2824859917163849,
+      "epoch": 0.29055690072639223,
+      "grad_norm": 0.1904715597629547,
+      "learning_rate": 4.97624123005063e-05,
+      "loss": 1.282959222793579,
+      "mean_token_accuracy": 0.6380399465560913,
+      "num_tokens": 2903815.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.274334877729416,
+      "epoch": 0.30024213075060535,
+      "grad_norm": 0.17454974353313446,
+      "learning_rate": 4.972452435550614e-05,
+      "loss": 1.2828162908554077,
+      "mean_token_accuracy": 0.6363882124423981,
+      "num_tokens": 3000656.0,
+      "step": 31
+    },
+    {
+      "entropy": 1.298419788479805,
+      "epoch": 0.3099273607748184,
+      "grad_norm": 0.16989658772945404,
+      "learning_rate": 4.968385131436222e-05,
+      "loss": 1.283800482749939,
+      "mean_token_accuracy": 0.6383039951324463,
+      "num_tokens": 3098537.0,
+      "step": 32
+    },
+    {
+      "entropy": 1.3138496726751328,
+      "epoch": 0.3196125907990315,
+      "grad_norm": 0.1963842809200287,
+      "learning_rate": 4.9640397758692715e-05,
+      "loss": 1.287667989730835,
+      "mean_token_accuracy": 0.6366405189037323,
+      "num_tokens": 3195923.0,
+      "step": 33
+    },
+    {
+      "entropy": 1.316459745168686,
+      "epoch": 0.32929782082324455,
+      "grad_norm": 0.17209197580814362,
+      "learning_rate": 4.9594168583327094e-05,
+      "loss": 1.2872288227081299,
+      "mean_token_accuracy": 0.6377927586436272,
+      "num_tokens": 3293424.0,
+      "step": 34
+    },
+    {
+      "entropy": 1.2688211798667908,
+      "epoch": 0.3389830508474576,
+      "grad_norm": 0.1710345298051834,
+      "learning_rate": 4.954516899575473e-05,
+      "loss": 1.2326550483703613,
+      "mean_token_accuracy": 0.6454800367355347,
+      "num_tokens": 3390314.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.275676041841507,
+      "epoch": 0.3486682808716707,
+      "grad_norm": 0.16850367188453674,
+      "learning_rate": 4.949340451553833e-05,
+      "loss": 1.2480475902557373,
+      "mean_token_accuracy": 0.644576296210289,
+      "num_tokens": 3485697.0,
+      "step": 36
+    },
+    {
+      "entropy": 1.2488422840833664,
+      "epoch": 0.3583535108958838,
+      "grad_norm": 0.17575675249099731,
+      "learning_rate": 4.943888097369216e-05,
+      "loss": 1.2449251413345337,
+      "mean_token_accuracy": 0.6482485681772232,
+      "num_tokens": 3583116.0,
+      "step": 37
+    },
+    {
+      "entropy": 1.20504991710186,
+      "epoch": 0.36803874092009686,
+      "grad_norm": 0.16634824872016907,
+      "learning_rate": 4.938160451202521e-05,
+      "loss": 1.210385799407959,
+      "mean_token_accuracy": 0.6547724008560181,
+      "num_tokens": 3678517.0,
+      "step": 38
+    },
+    {
+      "entropy": 1.236429587006569,
+      "epoch": 0.37772397094430993,
+      "grad_norm": 0.15360407531261444,
+      "learning_rate": 4.9321581582449365e-05,
+      "loss": 1.2314069271087646,
+      "mean_token_accuracy": 0.6504408493638039,
+      "num_tokens": 3775690.0,
+      "step": 39
+    },
+    {
+      "entropy": 1.2558289468288422,
+      "epoch": 0.387409200968523,
+      "grad_norm": 0.1626509577035904,
+      "learning_rate": 4.9258818946252624e-05,
+      "loss": 1.253812551498413,
+      "mean_token_accuracy": 0.6433829963207245,
+      "num_tokens": 3873239.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.2188849300146103,
+      "epoch": 0.39709443099273606,
+      "grad_norm": 0.16926692426204681,
+      "learning_rate": 4.9193323673337476e-05,
+      "loss": 1.2143971920013428,
+      "mean_token_accuracy": 0.6519596204161644,
+      "num_tokens": 3970507.0,
+      "step": 41
+    },
+    {
+      "entropy": 1.2296934127807617,
+      "epoch": 0.4067796610169492,
+      "grad_norm": 0.17094968259334564,
+      "learning_rate": 4.912510314142448e-05,
+      "loss": 1.2195372581481934,
+      "mean_token_accuracy": 0.6513471901416779,
+      "num_tokens": 4067145.0,
+      "step": 42
+    },
+    {
+      "entropy": 1.2168416231870651,
+      "epoch": 0.41646489104116224,
+      "grad_norm": 0.15936042368412018,
+      "learning_rate": 4.9054165035221236e-05,
+      "loss": 1.2123249769210815,
+      "mean_token_accuracy": 0.6524514406919479,
+      "num_tokens": 4165113.0,
+      "step": 43
+    },
+    {
+      "entropy": 1.1896271854639053,
+      "epoch": 0.4261501210653753,
+      "grad_norm": 0.15422837436199188,
+      "learning_rate": 4.898051734555676e-05,
+      "loss": 1.1957529783248901,
+      "mean_token_accuracy": 0.6586505621671677,
+      "num_tokens": 4262663.0,
+      "step": 44
+    },
+    {
+      "entropy": 1.2026645243167877,
+      "epoch": 0.4358353510895884,
+      "grad_norm": 0.16097040474414825,
+      "learning_rate": 4.890416836848127e-05,
+      "loss": 1.1992558240890503,
+      "mean_token_accuracy": 0.6544606015086174,
+      "num_tokens": 4359900.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.1843629777431488,
+      "epoch": 0.44552058111380144,
+      "grad_norm": 0.1597595363855362,
+      "learning_rate": 4.88251267043318e-05,
+      "loss": 1.1683578491210938,
+      "mean_token_accuracy": 0.6638541519641876,
+      "num_tokens": 4456985.0,
+      "step": 46
+    },
+    {
+      "entropy": 1.1679482907056808,
+      "epoch": 0.4552058111380145,
+      "grad_norm": 0.15203599631786346,
+      "learning_rate": 4.87434012567633e-05,
+      "loss": 1.1561784744262695,
+      "mean_token_accuracy": 0.6650881245732307,
+      "num_tokens": 4554634.0,
+      "step": 47
+    },
+    {
+      "entropy": 1.2105818390846252,
+      "epoch": 0.4648910411622276,
+      "grad_norm": 0.15597011148929596,
+      "learning_rate": 4.865900123174574e-05,
+      "loss": 1.1950294971466064,
+      "mean_token_accuracy": 0.6544563546776772,
+      "num_tokens": 4652249.0,
+      "step": 48
+    },
+    {
+      "entropy": 1.2046389728784561,
+      "epoch": 0.4745762711864407,
+      "grad_norm": 0.15933600068092346,
+      "learning_rate": 4.857193613652711e-05,
+      "loss": 1.183128833770752,
+      "mean_token_accuracy": 0.6564416959881783,
+      "num_tokens": 4748832.0,
+      "step": 49
+    },
+    {
+      "entropy": 1.2113090008497238,
+      "epoch": 0.48426150121065376,
+      "grad_norm": 0.15904678404331207,
+      "learning_rate": 4.848221577856243e-05,
+      "loss": 1.1915383338928223,
+      "mean_token_accuracy": 0.6567506939172745,
+      "num_tokens": 4845986.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1754019558429718,
+      "epoch": 0.4939467312348668,
+      "grad_norm": 0.15548165142536163,
+      "learning_rate": 4.838985026440905e-05,
+      "loss": 1.1645618677139282,
+      "mean_token_accuracy": 0.6654537692666054,
+      "num_tokens": 4942777.0,
+      "step": 51
+    },
+    {
+      "entropy": 1.1717564463615417,
+      "epoch": 0.5036319612590799,
+      "grad_norm": 0.16495537757873535,
+      "learning_rate": 4.829484999858815e-05,
+      "loss": 1.1674176454544067,
+      "mean_token_accuracy": 0.659358873963356,
+      "num_tokens": 5040326.0,
+      "step": 52
+    },
+    {
+      "entropy": 1.1637324541807175,
+      "epoch": 0.513317191283293,
+      "grad_norm": 0.16075873374938965,
+      "learning_rate": 4.819722568241274e-05,
+      "loss": 1.1621849536895752,
+      "mean_token_accuracy": 0.663130484521389,
+      "num_tokens": 5137838.0,
+      "step": 53
+    },
+    {
+      "entropy": 1.1703068912029266,
+      "epoch": 0.5230024213075061,
+      "grad_norm": 0.16048873960971832,
+      "learning_rate": 4.8096988312782174e-05,
+      "loss": 1.1667448282241821,
+      "mean_token_accuracy": 0.6606730446219444,
+      "num_tokens": 5234578.0,
+      "step": 54
+    },
+    {
+      "entropy": 1.1869463622570038,
+      "epoch": 0.5326876513317191,
+      "grad_norm": 0.1574316769838333,
+      "learning_rate": 4.799414918094347e-05,
+      "loss": 1.1867117881774902,
+      "mean_token_accuracy": 0.6562071144580841,
+      "num_tokens": 5332439.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.1833796501159668,
+      "epoch": 0.5423728813559322,
+      "grad_norm": 0.1627049446105957,
+      "learning_rate": 4.788871987121937e-05,
+      "loss": 1.1771697998046875,
+      "mean_token_accuracy": 0.6603464111685753,
+      "num_tokens": 5430102.0,
+      "step": 56
+    },
+    {
+      "entropy": 1.1720363795757294,
+      "epoch": 0.5520581113801453,
+      "grad_norm": 0.30378302931785583,
+      "learning_rate": 4.77807122597034e-05,
+      "loss": 1.1528563499450684,
+      "mean_token_accuracy": 0.6617102548480034,
+      "num_tokens": 5528264.0,
+      "step": 57
+    },
+    {
+      "entropy": 1.1582888960838318,
+      "epoch": 0.5617433414043583,
+      "grad_norm": 0.1613432615995407,
+      "learning_rate": 4.767013851292212e-05,
+      "loss": 1.1482983827590942,
+      "mean_token_accuracy": 0.6675111055374146,
+      "num_tokens": 5625491.0,
+      "step": 58
+    },
+    {
+      "entropy": 1.1704413145780563,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.15665417909622192,
+      "learning_rate": 4.755701108646463e-05,
+      "loss": 1.1476081609725952,
+      "mean_token_accuracy": 0.6660670340061188,
+      "num_tokens": 5723563.0,
+      "step": 59
+    },
+    {
+      "entropy": 1.1684788465499878,
+      "epoch": 0.5811138014527845,
+      "grad_norm": 0.1587826907634735,
+      "learning_rate": 4.744134272357948e-05,
+      "loss": 1.15497624874115,
+      "mean_token_accuracy": 0.662791483104229,
+      "num_tokens": 5821106.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1705324202775955,
+      "epoch": 0.5907990314769975,
+      "grad_norm": 0.1650344729423523,
+      "learning_rate": 4.732314645373921e-05,
+      "loss": 1.168868899345398,
+      "mean_token_accuracy": 0.659059427678585,
+      "num_tokens": 5918772.0,
+      "step": 61
+    },
+    {
+      "entropy": 1.1340993344783783,
+      "epoch": 0.6004842615012107,
+      "grad_norm": 0.15585103631019592,
+      "learning_rate": 4.7202435591172676e-05,
+      "loss": 1.1181915998458862,
+      "mean_token_accuracy": 0.6716931834816933,
+      "num_tokens": 6016758.0,
+      "step": 62
+    },
+    {
+      "entropy": 1.1135689467191696,
+      "epoch": 0.6101694915254238,
+      "grad_norm": 0.15927211940288544,
+      "learning_rate": 4.7079223733365237e-05,
+      "loss": 1.1100869178771973,
+      "mean_token_accuracy": 0.6781439930200577,
+      "num_tokens": 6113210.0,
+      "step": 63
+    },
+    {
+      "entropy": 1.1656545102596283,
+      "epoch": 0.6198547215496368,
+      "grad_norm": 0.1585935652256012,
+      "learning_rate": 4.6953524759527054e-05,
+      "loss": 1.1522451639175415,
+      "mean_token_accuracy": 0.664880745112896,
+      "num_tokens": 6210272.0,
+      "step": 64
+    },
+    {
+      "entropy": 1.1241111010313034,
+      "epoch": 0.6295399515738499,
+      "grad_norm": 0.15607106685638428,
+      "learning_rate": 4.6825352829029705e-05,
+      "loss": 1.1200485229492188,
+      "mean_token_accuracy": 0.671033151447773,
+      "num_tokens": 6307273.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.135048821568489,
+      "epoch": 0.639225181598063,
+      "grad_norm": 0.1598539501428604,
+      "learning_rate": 4.6694722379811185e-05,
+      "loss": 1.1302683353424072,
+      "mean_token_accuracy": 0.6700672060251236,
+      "num_tokens": 6404865.0,
+      "step": 66
+    },
+    {
+      "entropy": 1.1544559001922607,
+      "epoch": 0.648910411622276,
+      "grad_norm": 0.16644105315208435,
+      "learning_rate": 4.656164812674951e-05,
+      "loss": 1.1450002193450928,
+      "mean_token_accuracy": 0.6640813648700714,
+      "num_tokens": 6501650.0,
+      "step": 67
+    },
+    {
+      "entropy": 1.132233053445816,
+      "epoch": 0.6585956416464891,
+      "grad_norm": 0.16045193374156952,
+      "learning_rate": 4.642614506000523e-05,
+      "loss": 1.121936559677124,
+      "mean_token_accuracy": 0.6716863736510277,
+      "num_tokens": 6599028.0,
+      "step": 68
+    },
+    {
+      "entropy": 1.1332774758338928,
+      "epoch": 0.6682808716707022,
+      "grad_norm": 0.16355212032794952,
+      "learning_rate": 4.628822844333278e-05,
+      "loss": 1.1154568195343018,
+      "mean_token_accuracy": 0.6706111207604408,
+      "num_tokens": 6695913.0,
+      "step": 69
+    },
+    {
+      "entropy": 1.117134228348732,
+      "epoch": 0.6779661016949152,
+      "grad_norm": 0.1609818935394287,
+      "learning_rate": 4.614791381236115e-05,
+      "loss": 1.108933925628662,
+      "mean_token_accuracy": 0.6739098057150841,
+      "num_tokens": 6793755.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.123718187212944,
+      "epoch": 0.6876513317191283,
+      "grad_norm": 0.16711370646953583,
+      "learning_rate": 4.6005216972843864e-05,
+      "loss": 1.1205769777297974,
+      "mean_token_accuracy": 0.6695492789149284,
+      "num_tokens": 6889869.0,
+      "step": 71
+    },
+    {
+      "entropy": 1.1330177187919617,
+      "epoch": 0.6973365617433414,
+      "grad_norm": 0.15766654908657074,
+      "learning_rate": 4.5860153998878494e-05,
+      "loss": 1.136279582977295,
+      "mean_token_accuracy": 0.6664912849664688,
+      "num_tokens": 6987242.0,
+      "step": 72
+    },
+    {
+      "entropy": 1.1289773285388947,
+      "epoch": 0.7070217917675545,
+      "grad_norm": 0.167424738407135,
+      "learning_rate": 4.571274123109606e-05,
+      "loss": 1.1257307529449463,
+      "mean_token_accuracy": 0.6683443114161491,
+      "num_tokens": 7083683.0,
+      "step": 73
+    },
+    {
+      "entropy": 1.1089464724063873,
+      "epoch": 0.7167070217917676,
+      "grad_norm": 0.29150310158729553,
+      "learning_rate": 4.5562995274820284e-05,
+      "loss": 1.0986303091049194,
+      "mean_token_accuracy": 0.6785000637173653,
+      "num_tokens": 7180007.0,
+      "step": 74
+    },
+    {
+      "entropy": 1.1168130189180374,
+      "epoch": 0.7263922518159807,
+      "grad_norm": 0.1623908281326294,
+      "learning_rate": 4.541093299819714e-05,
+      "loss": 1.1109479665756226,
+      "mean_token_accuracy": 0.6751653999090195,
+      "num_tokens": 7276880.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.1238265335559845,
+      "epoch": 0.7360774818401937,
+      "grad_norm": 0.16212861239910126,
+      "learning_rate": 4.5256571530294664e-05,
+      "loss": 1.104738473892212,
+      "mean_token_accuracy": 0.6755917444825172,
+      "num_tokens": 7374051.0,
+      "step": 76
+    },
+    {
+      "entropy": 1.1079749763011932,
+      "epoch": 0.7457627118644068,
+      "grad_norm": 0.15648601949214935,
+      "learning_rate": 4.5099928259173516e-05,
+      "loss": 1.1000884771347046,
+      "mean_token_accuracy": 0.6749309673905373,
+      "num_tokens": 7471906.0,
+      "step": 77
+    },
+    {
+      "entropy": 1.12865149974823,
+      "epoch": 0.7554479418886199,
+      "grad_norm": 0.15941032767295837,
+      "learning_rate": 4.4941020829928245e-05,
+      "loss": 1.1193838119506836,
+      "mean_token_accuracy": 0.6708522811532021,
+      "num_tokens": 7569981.0,
+      "step": 78
+    },
+    {
+      "entropy": 1.1111949235200882,
+      "epoch": 0.7651331719128329,
+      "grad_norm": 0.16523951292037964,
+      "learning_rate": 4.477986714269972e-05,
+      "loss": 1.1221985816955566,
+      "mean_token_accuracy": 0.6717454493045807,
+      "num_tokens": 7663566.0,
+      "step": 79
+    },
+    {
+      "entropy": 1.100884273648262,
+      "epoch": 0.774818401937046,
+      "grad_norm": 0.16432514786720276,
+      "learning_rate": 4.4616485350658685e-05,
+      "loss": 1.104296088218689,
+      "mean_token_accuracy": 0.6767150536179543,
+      "num_tokens": 7761120.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.1229810565710068,
+      "epoch": 0.784503631961259,
+      "grad_norm": 0.17101751267910004,
+      "learning_rate": 4.445089385796099e-05,
+      "loss": 1.1202152967453003,
+      "mean_token_accuracy": 0.6708082109689713,
+      "num_tokens": 7859307.0,
+      "step": 81
+    },
+    {
+      "entropy": 1.134630709886551,
+      "epoch": 0.7941888619854721,
+      "grad_norm": 0.1734648197889328,
+      "learning_rate": 4.4283111317674374e-05,
+      "loss": 1.1201170682907104,
+      "mean_token_accuracy": 0.6716241240501404,
+      "num_tokens": 7956863.0,
+      "step": 82
+    },
+    {
+      "entropy": 1.1259192377328873,
+      "epoch": 0.8038740920096852,
+      "grad_norm": 0.16475871205329895,
+      "learning_rate": 4.4113156629677316e-05,
+      "loss": 1.1068096160888672,
+      "mean_token_accuracy": 0.674552284181118,
+      "num_tokens": 8053100.0,
+      "step": 83
+    },
+    {
+      "entropy": 1.0983240008354187,
+      "epoch": 0.8135593220338984,
+      "grad_norm": 0.17292368412017822,
+      "learning_rate": 4.394104893853007e-05,
+      "loss": 1.0763158798217773,
+      "mean_token_accuracy": 0.6806622296571732,
+      "num_tokens": 8151004.0,
+      "step": 84
+    },
+    {
+      "entropy": 1.1288987547159195,
+      "epoch": 0.8232445520581114,
+      "grad_norm": 0.16203227639198303,
+      "learning_rate": 4.3766807631318106e-05,
+      "loss": 1.1195769309997559,
+      "mean_token_accuracy": 0.671344555914402,
+      "num_tokens": 8248668.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.1410782486200333,
+      "epoch": 0.8329297820823245,
+      "grad_norm": 0.16533830761909485,
+      "learning_rate": 4.359045233546827e-05,
+      "loss": 1.1256823539733887,
+      "mean_token_accuracy": 0.6698467060923576,
+      "num_tokens": 8345497.0,
+      "step": 86
+    },
+    {
+      "entropy": 1.0999096632003784,
+      "epoch": 0.8426150121065376,
+      "grad_norm": 0.17075778543949127,
+      "learning_rate": 4.341200291653781e-05,
+      "loss": 1.0872827768325806,
+      "mean_token_accuracy": 0.6762653961777687,
+      "num_tokens": 8443405.0,
+      "step": 87
+    },
+    {
+      "entropy": 1.0918122231960297,
+      "epoch": 0.8523002421307506,
+      "grad_norm": 0.1702580451965332,
+      "learning_rate": 4.323147947597666e-05,
+      "loss": 1.0806095600128174,
+      "mean_token_accuracy": 0.6800353080034256,
+      "num_tokens": 8540868.0,
+      "step": 88
+    },
+    {
+      "entropy": 1.0941280871629715,
+      "epoch": 0.8619854721549637,
+      "grad_norm": 0.16791851818561554,
+      "learning_rate": 4.3048902348863116e-05,
+      "loss": 1.0981943607330322,
+      "mean_token_accuracy": 0.6758224442601204,
+      "num_tokens": 8638186.0,
+      "step": 89
+    },
+    {
+      "entropy": 1.0951679199934006,
+      "epoch": 0.8716707021791767,
+      "grad_norm": 0.1697986125946045,
+      "learning_rate": 4.2864292101613136e-05,
+      "loss": 1.0987718105316162,
+      "mean_token_accuracy": 0.6747072339057922,
+      "num_tokens": 8735354.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1223643571138382,
+      "epoch": 0.8813559322033898,
+      "grad_norm": 0.2108999490737915,
+      "learning_rate": 4.267766952966369e-05,
+      "loss": 1.1219862699508667,
+      "mean_token_accuracy": 0.6695899069309235,
+      "num_tokens": 8832051.0,
+      "step": 91
+    },
+    {
+      "entropy": 1.1099307239055634,
+      "epoch": 0.8910411622276029,
+      "grad_norm": 0.18003661930561066,
+      "learning_rate": 4.248905565513023e-05,
+      "loss": 1.0990370512008667,
+      "mean_token_accuracy": 0.6750800833106041,
+      "num_tokens": 8929471.0,
+      "step": 92
+    },
+    {
+      "entropy": 1.1097175031900406,
+      "epoch": 0.9007263922518159,
+      "grad_norm": 0.18473289906978607,
+      "learning_rate": 4.229847172443866e-05,
+      "loss": 1.0890154838562012,
+      "mean_token_accuracy": 0.6764732897281647,
+      "num_tokens": 9026545.0,
+      "step": 93
+    },
+    {
+      "entropy": 1.1048423945903778,
+      "epoch": 0.910411622276029,
+      "grad_norm": 0.17208412289619446,
+      "learning_rate": 4.210593920593201e-05,
+      "loss": 1.1022264957427979,
+      "mean_token_accuracy": 0.6741294264793396,
+      "num_tokens": 9124261.0,
+      "step": 94
+    },
+    {
+      "entropy": 1.0943758487701416,
+      "epoch": 0.9200968523002422,
+      "grad_norm": 0.18163490295410156,
+      "learning_rate": 4.191147978745218e-05,
+      "loss": 1.0901988744735718,
+      "mean_token_accuracy": 0.6788006126880646,
+      "num_tokens": 9220673.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.1178465783596039,
+      "epoch": 0.9297820823244553,
+      "grad_norm": 0.1731623411178589,
+      "learning_rate": 4.171511537389684e-05,
+      "loss": 1.1111767292022705,
+      "mean_token_accuracy": 0.6736986339092255,
+      "num_tokens": 9318321.0,
+      "step": 96
+    },
+    {
+      "entropy": 1.0975633561611176,
+      "epoch": 0.9394673123486683,
+      "grad_norm": 0.18208175897598267,
+      "learning_rate": 4.151686808475204e-05,
+      "loss": 1.0915802717208862,
+      "mean_token_accuracy": 0.6765245422720909,
+      "num_tokens": 9415394.0,
+      "step": 97
+    },
+    {
+      "entropy": 1.114044964313507,
+      "epoch": 0.9491525423728814,
+      "grad_norm": 0.17058847844600677,
+      "learning_rate": 4.131676025160047e-05,
+      "loss": 1.1074469089508057,
+      "mean_token_accuracy": 0.6720306798815727,
+      "num_tokens": 9512880.0,
+      "step": 98
+    },
+    {
+      "entropy": 1.07672019302845,
+      "epoch": 0.9588377723970944,
+      "grad_norm": 0.18478929996490479,
+      "learning_rate": 4.111481441560598e-05,
+      "loss": 1.0674448013305664,
+      "mean_token_accuracy": 0.6838654279708862,
+      "num_tokens": 9606053.0,
+      "step": 99
+    },
+    {
+      "entropy": 1.0935327857732773,
+      "epoch": 0.9685230024213075,
+      "grad_norm": 0.17510797083377838,
+      "learning_rate": 4.091105332497439e-05,
+      "loss": 1.0954351425170898,
+      "mean_token_accuracy": 0.6775561571121216,
+      "num_tokens": 9702569.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.0855822414159775,
+      "epoch": 0.9782082324455206,
+      "grad_norm": 0.18510136008262634,
+      "learning_rate": 4.070549993239106e-05,
+      "loss": 1.0844038724899292,
+      "mean_token_accuracy": 0.681103028357029,
+      "num_tokens": 9799976.0,
+      "step": 101
+    },
+    {
+      "entropy": 1.1072624325752258,
+      "epoch": 0.9878934624697336,
+      "grad_norm": 0.2109905481338501,
+      "learning_rate": 4.049817739243532e-05,
+      "loss": 1.1046338081359863,
+      "mean_token_accuracy": 0.6760703399777412,
+      "num_tokens": 9895575.0,
+      "step": 102
+    },
+    {
+      "entropy": 1.0761565268039703,
+      "epoch": 0.9975786924939467,
+      "grad_norm": 0.18559105694293976,
+      "learning_rate": 4.028910905897229e-05,
+      "loss": 1.0674762725830078,
+      "mean_token_accuracy": 0.6826195642352104,
+      "num_tokens": 9992394.0,
+      "step": 103
+    },
+    {
+      "entropy": 1.1065305471420288,
+      "epoch": 1.0,
+      "grad_norm": 0.3376278877258301,
+      "learning_rate": 4.007831848252211e-05,
+      "loss": 1.09256911277771,
+      "mean_token_accuracy": 0.678756445646286,
+      "num_tokens": 10016415.0,
+      "step": 104
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 312,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.4089024450435482e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}