Add 11 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+training-metrics.png filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,99 @@
+---
+base_model: meta-llama/Llama-4-Scout-17B-16E-Instruct
+library_name: peft
+license: other
+tags:
+  - lora
+  - peft
+  - adapter
+  - adaption
+---
+
+# adaption_crop_disease_reasoning
+
+## Model Training
+
+A LORA adapter for `meta-llama/Llama-4-Scout-17B-16E-Instruct`. This model was trained with SFT using [Adaption](https://adaptionlabs.ai)'s AutoScientist on the crop_disease_reasoning dataset.
+
+
+![Training metrics](training-metrics.png)
+
+### AutoScientist Config
+
+```json
+{
+  "job_id": "eb780706-a44d-459d-8ce6-c666e459dc76",
+  "training_experiment_id": "9aa3a4bd-67ac-4a69-afc1-06cc11b3320b",
+  "original_model_name": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
+  "trained_model_name": "adaption_crop_disease_reasoning",
+  "training_method": "sft",
+  "training_type": "lora",
+  "data_format": "chat",
+  "hyperparams": {
+    "lora": "true",
+    "lora_r": 64,
+    "n_evals": 5,
+    "n_epochs": 4,
+    "batch_size": "max",
+    "lora_alpha": 128,
+    "lora_dropout": 0,
+    "min_lr_ratio": 0.1,
+    "warmup_ratio": 0.03,
+    "weight_decay": 0.02,
+    "learning_rate": 0.0001,
+    "max_grad_norm": 1,
+    "base_model_size": "109B",
+    "train_on_inputs": "false",
+    "training_method": "sft",
+    "lr_scheduler_type": "cosine",
+    "scheduler_num_cycles": 0.5,
+    "lora_trainable_modules": "k_proj,o_proj,q_proj,v_proj,shared_expert.gate_proj,shared_expert.up_proj,shared_expert.down_proj,feed_forward.gate_proj,feed_forward.up_proj,feed_forward.down_proj"
+  }
+}
+```
+
+## Training Data
+
+The model was trained on 34,323 rows of adapted data with the following domain distribution: agriculture (100%).
+
+## Model Evaluation
+
+The model was evaluated on an in-distribution held-out test set as well as a broader domain-specific test set to measure generalization.
+
+
+![Win rates](win-rates.png)
+
+
+## How to use
+
+```bash
+pip install torch transformers peft
+```
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+BASE = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
+ADAPTER = "<this-repo-id>"
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+dtype = torch.float32 if device == "cpu" else torch.bfloat16
+
+base = AutoModelForCausalLM.from_pretrained(BASE, dtype=dtype).to(device)
+model = PeftModel.from_pretrained(base, ADAPTER)
+# Optional: merge the LoRA weights into the base for faster inference
+model = model.merge_and_unload()
+model.eval()
+
+tokenizer = AutoTokenizer.from_pretrained(BASE)
+messages = [{"role": "user", "content": "Hello!"}]
+text = tokenizer.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True)
+inputs = tokenizer(text, return_tensors="pt").to(device)
+
+with torch.inference_mode():
+    out = model.generate(**inputs, max_new_tokens=512)
+print(tokenizer.decode(out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True))
+```

adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "togethercomputer/Llama-4-Scout-17B-16E-Instruct_bnb_4bit",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": [],
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "o_proj",
+    "feed_forward.up_proj",
+    "feed_forward.down_proj",
+    "v_proj",
+    "shared_expert.down_proj",
+    "shared_expert.gate_proj",
+    "k_proj",
+    "shared_expert.up_proj",
+    "feed_forward.gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7a557e4c368cce5c8d1cc1d896a64b1105562ce381d738e981d2008858153b0
+size 893484120

chat_template.jinja

@@ -0,0 +1,123 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}    
+    {%- if messages[0]['content'] is string %}
+        {%- set system_message = messages[0]['content']|trim %}
+    {%- else %}
+        {#- FIXME: The processor requires an array, always. #}
+        {%- set system_message = messages[0]['content'][0]['text']|trim %}
+    {%- endif %}
+    {%- set messages = messages[1:] %}
+    {%- set user_supplied_system_message = true %}
+{%- else %}
+    {%- set system_message = "" %}
+    {%- set user_supplied_system_message = false %}
+{%- endif %}
+
+{#- System message if the user supplied one #}
+{%- if user_supplied_system_message %}
+    {{- "<|header_start|>system<|header_end|>\n\n" }}
+    {%- if tools is not none %}
+        {{- "Environment: ipython\n" }}
+    {%- endif %}
+    {%- if tools is not none and not tools_in_user_message %}
+        {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+        {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+        {{- "Do not use variables.\n\n" }}
+        {%- for t in tools %}
+            {{- t | tojson(indent=4) }}
+            {{- "\n\n" }}
+        {%- endfor %}
+    {%- endif %}
+    {{- system_message }}
+    {{- "<|eot|>" }}
+{%- endif %}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|header_start|>user<|header_end|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+    {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
+        {%- if message['content'] is string %}
+            {{- message['content'] }}
+        {%- else %}
+            {%- for content in message['content'] %}
+                {%- if content['type'] == 'image' %}
+                    {{- '<|image|>' }}
+                {%- elif content['type'] == 'text' %}
+                    {{- content['text'] }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- "<|eot|>" }}
+    {%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
+       {{- '<|header_start|>assistant<|header_end|>\n\n' -}}
+       {{- '<|python_start|>' }}
+        {%- if message['content'] is string %}
+            {{- message['content'] }}
+        {%- else %}
+            {%- for content in message['content'] %}
+                {%- if content['type'] == 'image' %}
+                    {{- '<|image|>' }}
+                {%- elif content['type'] == 'text' %}
+                    {{- content['text'] }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+       {{- '<|python_end|>' }}
+        {%- for tool_call in message.tool_calls %}
+           {{- '{"name": "' + tool_call.function.name + '", ' }}
+           {{- '"parameters": ' }}
+           {{- tool_call.function.arguments | tojson }}
+           {{- "}" }}
+        {%- endfor %}
+       {{- "<|eot|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|header_start|>ipython<|header_end|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|header_start|>assistant<|header_end|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,215 @@
+{
+  "architectures": [
+    "Llama4ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_chunk_size": 8192,
+  "attention_dropout": 0.0,
+  "attn_scale": 0.1,
+  "attn_temperature_tuning": true,
+  "bos_token_id": 200000,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 200008,
+  "floor_scale": 8192,
+  "for_llm_compressor": false,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "interleave_moe_layer_step": 1,
+  "intermediate_size": 8192,
+  "intermediate_size_mlp": 16384,
+  "layer_types": [
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "chunked_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 10485760,
+  "model_type": "llama4_text",
+  "moe_layers": [
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47
+  ],
+  "no_rope_layer_interval": 4,
+  "no_rope_layers": [
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0
+  ],
+  "num_attention_heads": 40,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "num_local_experts": 16,
+  "output_router_logits": false,
+  "pad_token_id": 200018,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": false,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 16.0,
+    "high_freq_factor": 1.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.10.1",
+  "use_cache": false,
+  "use_qk_norm": true,
+  "vocab_size": 202048,
+  "torch_dtype": "bfloat16"
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|begin_of_text|>",
+  "eos_token": "<|eot|>",
+  "pad_token": "<|finetune_right_pad|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:172c9eb4beafc72601690da3ccfcede5c2e6806a8d5ec1fca33e22acea8023a4
+size 27948578

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|eot|>",
+  "is_local": false,
+  "local_files_only": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 10485760,
+  "pad_token": "<|finetune_right_pad|>",
+  "padding_side": "right",
+  "processor_class": "Llama4Processor",
+  "tokenizer_class": "TokenizersBackend"
+}

trainer_state.json

@@ -0,0 +1,1642 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 44,
+  "global_step": 224,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017857142857142856,
+      "grad_norm": 0.31462591886520386,
+      "learning_rate": 0.0,
+      "loss": 1.7392578125,
+      "step": 1
+    },
+    {
+      "epoch": 0.03571428571428571,
+      "grad_norm": 0.2711670398712158,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 1.6328125,
+      "step": 2
+    },
+    {
+      "epoch": 0.05357142857142857,
+      "grad_norm": 0.30870506167411804,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 1.6630859375,
+      "step": 3
+    },
+    {
+      "epoch": 0.07142857142857142,
+      "grad_norm": 0.28610464930534363,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 1.658203125,
+      "step": 4
+    },
+    {
+      "epoch": 0.08928571428571429,
+      "grad_norm": 0.20997090637683868,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 1.6162109375,
+      "step": 5
+    },
+    {
+      "epoch": 0.10714285714285714,
+      "grad_norm": 0.1340269297361374,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 1.5986328125,
+      "step": 6
+    },
+    {
+      "epoch": 0.125,
+      "grad_norm": 0.11159807443618774,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 1.490234375,
+      "step": 7
+    },
+    {
+      "epoch": 0.14285714285714285,
+      "grad_norm": 0.15790338814258575,
+      "learning_rate": 0.0001,
+      "loss": 1.48046875,
+      "step": 8
+    },
+    {
+      "epoch": 0.16071428571428573,
+      "grad_norm": 0.21403393149375916,
+      "learning_rate": 9.99952842018222e-05,
+      "loss": 1.4365234375,
+      "step": 9
+    },
+    {
+      "epoch": 0.17857142857142858,
+      "grad_norm": 0.18980804085731506,
+      "learning_rate": 9.998113779567773e-05,
+      "loss": 1.4560546875,
+      "step": 10
+    },
+    {
+      "epoch": 0.19642857142857142,
+      "grad_norm": 0.11710498481988907,
+      "learning_rate": 9.995756374652646e-05,
+      "loss": 1.3837890625,
+      "step": 11
+    },
+    {
+      "epoch": 0.21428571428571427,
+      "grad_norm": 0.08485613018274307,
+      "learning_rate": 9.992456699527763e-05,
+      "loss": 1.4248046875,
+      "step": 12
+    },
+    {
+      "epoch": 0.23214285714285715,
+      "grad_norm": 0.07681010663509369,
+      "learning_rate": 9.988215445775436e-05,
+      "loss": 1.404296875,
+      "step": 13
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.06381040811538696,
+      "learning_rate": 9.983033502324401e-05,
+      "loss": 1.31787109375,
+      "step": 14
+    },
+    {
+      "epoch": 0.26785714285714285,
+      "grad_norm": 0.2749612033367157,
+      "learning_rate": 9.976911955263529e-05,
+      "loss": 1.1463623046875,
+      "step": 15
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 0.07103897631168365,
+      "learning_rate": 9.969852087614172e-05,
+      "loss": 1.30078125,
+      "step": 16
+    },
+    {
+      "epoch": 0.30357142857142855,
+      "grad_norm": 0.06604181230068207,
+      "learning_rate": 9.961855379061266e-05,
+      "loss": 1.24951171875,
+      "step": 17
+    },
+    {
+      "epoch": 0.32142857142857145,
+      "grad_norm": 0.06762582063674927,
+      "learning_rate": 9.952923505643191e-05,
+      "loss": 1.380859375,
+      "step": 18
+    },
+    {
+      "epoch": 0.3392857142857143,
+      "grad_norm": 0.0505656860768795,
+      "learning_rate": 9.9430583394005e-05,
+      "loss": 1.318359375,
+      "step": 19
+    },
+    {
+      "epoch": 0.35714285714285715,
+      "grad_norm": 0.04348813742399216,
+      "learning_rate": 9.932261947983547e-05,
+      "loss": 1.296875,
+      "step": 20
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 0.04398291930556297,
+      "learning_rate": 9.92053659421913e-05,
+      "loss": 1.3359375,
+      "step": 21
+    },
+    {
+      "epoch": 0.39285714285714285,
+      "grad_norm": 0.043604083359241486,
+      "learning_rate": 9.907884735636226e-05,
+      "loss": 1.265625,
+      "step": 22
+    },
+    {
+      "epoch": 0.4107142857142857,
+      "grad_norm": 0.04494720697402954,
+      "learning_rate": 9.894309023950905e-05,
+      "loss": 1.14459228515625,
+      "step": 23
+    },
+    {
+      "epoch": 0.42857142857142855,
+      "grad_norm": 0.0465690903365612,
+      "learning_rate": 9.879812304510566e-05,
+      "loss": 1.2802734375,
+      "step": 24
+    },
+    {
+      "epoch": 0.44642857142857145,
+      "grad_norm": 0.03763221576809883,
+      "learning_rate": 9.864397615697568e-05,
+      "loss": 1.251953125,
+      "step": 25
+    },
+    {
+      "epoch": 0.4642857142857143,
+      "grad_norm": 0.04064473882317543,
+      "learning_rate": 9.848068188292421e-05,
+      "loss": 1.2607421875,
+      "step": 26
+    },
+    {
+      "epoch": 0.48214285714285715,
+      "grad_norm": 0.03627771884202957,
+      "learning_rate": 9.830827444796634e-05,
+      "loss": 1.1982421875,
+      "step": 27
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.03516250103712082,
+      "learning_rate": 9.812678998715397e-05,
+      "loss": 1.20654296875,
+      "step": 28
+    },
+    {
+      "epoch": 0.5178571428571429,
+      "grad_norm": 0.03213035315275192,
+      "learning_rate": 9.793626653800219e-05,
+      "loss": 1.1690673828125,
+      "step": 29
+    },
+    {
+      "epoch": 0.5357142857142857,
+      "grad_norm": 0.035089440643787384,
+      "learning_rate": 9.773674403251701e-05,
+      "loss": 1.24609375,
+      "step": 30
+    },
+    {
+      "epoch": 0.5535714285714286,
+      "grad_norm": 0.03564055263996124,
+      "learning_rate": 9.752826428882581e-05,
+      "loss": 1.208984375,
+      "step": 31
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.03575805202126503,
+      "learning_rate": 9.731087100241291e-05,
+      "loss": 1.2265625,
+      "step": 32
+    },
+    {
+      "epoch": 0.5892857142857143,
+      "grad_norm": 0.035467199981212616,
+      "learning_rate": 9.708460973696108e-05,
+      "loss": 1.216796875,
+      "step": 33
+    },
+    {
+      "epoch": 0.6071428571428571,
+      "grad_norm": 0.03591489419341087,
+      "learning_rate": 9.684952791480208e-05,
+      "loss": 1.2275390625,
+      "step": 34
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 0.03851291164755821,
+      "learning_rate": 9.660567480697717e-05,
+      "loss": 1.2255859375,
+      "step": 35
+    },
+    {
+      "epoch": 0.6428571428571429,
+      "grad_norm": 0.03599183261394501,
+      "learning_rate": 9.635310152291039e-05,
+      "loss": 1.2216796875,
+      "step": 36
+    },
+    {
+      "epoch": 0.6607142857142857,
+      "grad_norm": 0.03200181946158409,
+      "learning_rate": 9.609186099969655e-05,
+      "loss": 1.17236328125,
+      "step": 37
+    },
+    {
+      "epoch": 0.6785714285714286,
+      "grad_norm": 0.032688844949007034,
+      "learning_rate": 9.582200799100601e-05,
+      "loss": 1.1484375,
+      "step": 38
+    },
+    {
+      "epoch": 0.6964285714285714,
+      "grad_norm": 0.03375864028930664,
+      "learning_rate": 9.554359905560886e-05,
+      "loss": 1.20703125,
+      "step": 39
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 0.04003170132637024,
+      "learning_rate": 9.525669254552067e-05,
+      "loss": 1.1796875,
+      "step": 40
+    },
+    {
+      "epoch": 0.7321428571428571,
+      "grad_norm": 0.038663070648908615,
+      "learning_rate": 9.496134859377242e-05,
+      "loss": 1.208984375,
+      "step": 41
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.031495656818151474,
+      "learning_rate": 9.46576291018072e-05,
+      "loss": 1.05224609375,
+      "step": 42
+    },
+    {
+      "epoch": 0.7678571428571429,
+      "grad_norm": 0.04218436777591705,
+      "learning_rate": 9.43455977265062e-05,
+      "loss": 1.1669921875,
+      "step": 43
+    },
+    {
+      "epoch": 0.7857142857142857,
+      "grad_norm": 0.032447610050439835,
+      "learning_rate": 9.402531986684682e-05,
+      "loss": 1.12548828125,
+      "step": 44
+    },
+    {
+      "epoch": 0.8035714285714286,
+      "grad_norm": 0.03663216531276703,
+      "learning_rate": 9.369686265019557e-05,
+      "loss": 1.142578125,
+      "step": 45
+    },
+    {
+      "epoch": 0.8214285714285714,
+      "grad_norm": 0.03747383505105972,
+      "learning_rate": 9.336029491823884e-05,
+      "loss": 1.1171875,
+      "step": 46
+    },
+    {
+      "epoch": 0.8392857142857143,
+      "grad_norm": 0.032268162816762924,
+      "learning_rate": 9.301568721255431e-05,
+      "loss": 1.142578125,
+      "step": 47
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 0.04622168838977814,
+      "learning_rate": 9.266311175982595e-05,
+      "loss": 1.1728515625,
+      "step": 48
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "eval_loss": 1.14892578125,
+      "eval_runtime": 5.5238,
+      "eval_samples_per_second": 0.543,
+      "eval_steps_per_second": 0.181,
+      "step": 48
+    },
+    {
+      "epoch": 0.875,
+      "grad_norm": 0.039248574525117874,
+      "learning_rate": 9.230264245670615e-05,
+      "loss": 1.150390625,
+      "step": 49
+    },
+    {
+      "epoch": 0.8928571428571429,
+      "grad_norm": 0.04478568583726883,
+      "learning_rate": 9.193435485432745e-05,
+      "loss": 1.1494140625,
+      "step": 50
+    },
+    {
+      "epoch": 0.9107142857142857,
+      "grad_norm": 0.03433000668883324,
+      "learning_rate": 9.155832614246782e-05,
+      "loss": 1.119140625,
+      "step": 51
+    },
+    {
+      "epoch": 0.9285714285714286,
+      "grad_norm": 0.033498235046863556,
+      "learning_rate": 9.11746351333723e-05,
+      "loss": 1.12451171875,
+      "step": 52
+    },
+    {
+      "epoch": 0.9464285714285714,
+      "grad_norm": 0.04067613556981087,
+      "learning_rate": 9.07833622452348e-05,
+      "loss": 1.1171875,
+      "step": 53
+    },
+    {
+      "epoch": 0.9642857142857143,
+      "grad_norm": 0.04021085798740387,
+      "learning_rate": 9.038458948534298e-05,
+      "loss": 1.1357421875,
+      "step": 54
+    },
+    {
+      "epoch": 0.9821428571428571,
+      "grad_norm": 0.035040102899074554,
+      "learning_rate": 8.997840043289034e-05,
+      "loss": 1.11572265625,
+      "step": 55
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.03852618858218193,
+      "learning_rate": 8.956488022145886e-05,
+      "loss": 1.1337890625,
+      "step": 56
+    },
+    {
+      "epoch": 1.0178571428571428,
+      "grad_norm": 0.04005853086709976,
+      "learning_rate": 8.914411552117559e-05,
+      "loss": 1.12109375,
+      "step": 57
+    },
+    {
+      "epoch": 1.0357142857142858,
+      "grad_norm": 0.03370766341686249,
+      "learning_rate": 8.871619452054753e-05,
+      "loss": 1.125,
+      "step": 58
+    },
+    {
+      "epoch": 1.0535714285714286,
+      "grad_norm": 0.03593919426202774,
+      "learning_rate": 8.828120690797799e-05,
+      "loss": 1.1064453125,
+      "step": 59
+    },
+    {
+      "epoch": 1.0714285714285714,
+      "grad_norm": 0.04067913070321083,
+      "learning_rate": 8.783924385296883e-05,
+      "loss": 1.08447265625,
+      "step": 60
+    },
+    {
+      "epoch": 1.0892857142857142,
+      "grad_norm": 0.03520175814628601,
+      "learning_rate": 8.739039798701195e-05,
+      "loss": 1.125,
+      "step": 61
+    },
+    {
+      "epoch": 1.1071428571428572,
+      "grad_norm": 0.04327214136719704,
+      "learning_rate": 8.693476338417483e-05,
+      "loss": 1.1181640625,
+      "step": 62
+    },
+    {
+      "epoch": 1.125,
+      "grad_norm": 0.03611943870782852,
+      "learning_rate": 8.64724355413832e-05,
+      "loss": 1.087890625,
+      "step": 63
+    },
+    {
+      "epoch": 1.1428571428571428,
+      "grad_norm": 0.03562260419130325,
+      "learning_rate": 8.600351135840589e-05,
+      "loss": 1.1181640625,
+      "step": 64
+    },
+    {
+      "epoch": 1.1607142857142858,
+      "grad_norm": 0.03648838773369789,
+      "learning_rate": 8.552808911754551e-05,
+      "loss": 1.08447265625,
+      "step": 65
+    },
+    {
+      "epoch": 1.1785714285714286,
+      "grad_norm": 0.03709696978330612,
+      "learning_rate": 8.504626846303918e-05,
+      "loss": 1.103515625,
+      "step": 66
+    },
+    {
+      "epoch": 1.1964285714285714,
+      "grad_norm": 0.034074269235134125,
+      "learning_rate": 8.455815038017428e-05,
+      "loss": 1.0595703125,
+      "step": 67
+    },
+    {
+      "epoch": 1.2142857142857142,
+      "grad_norm": 0.04193117842078209,
+      "learning_rate": 8.40638371741226e-05,
+      "loss": 1.068359375,
+      "step": 68
+    },
+    {
+      "epoch": 1.2321428571428572,
+      "grad_norm": 0.03742880001664162,
+      "learning_rate": 8.356343244849821e-05,
+      "loss": 1.08251953125,
+      "step": 69
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 0.03593381866812706,
+      "learning_rate": 8.305704108364301e-05,
+      "loss": 1.0791015625,
+      "step": 70
+    },
+    {
+      "epoch": 1.2678571428571428,
+      "grad_norm": 0.21050015091896057,
+      "learning_rate": 8.254476921464484e-05,
+      "loss": 0.91876220703125,
+      "step": 71
+    },
+    {
+      "epoch": 1.2857142857142856,
+      "grad_norm": 0.03813980519771576,
+      "learning_rate": 8.20267242090924e-05,
+      "loss": 1.05322265625,
+      "step": 72
+    },
+    {
+      "epoch": 1.3035714285714286,
+      "grad_norm": 0.03416725993156433,
+      "learning_rate": 8.150301464457205e-05,
+      "loss": 1.0185546875,
+      "step": 73
+    },
+    {
+      "epoch": 1.3214285714285714,
+      "grad_norm": 0.04501667991280556,
+      "learning_rate": 8.097375028591093e-05,
+      "loss": 1.119140625,
+      "step": 74
+    },
+    {
+      "epoch": 1.3392857142857144,
+      "grad_norm": 0.039226219058036804,
+      "learning_rate": 8.043904206217118e-05,
+      "loss": 1.0712890625,
+      "step": 75
+    },
+    {
+      "epoch": 1.3571428571428572,
+      "grad_norm": 0.03748096153140068,
+      "learning_rate": 7.98990020434002e-05,
+      "loss": 1.05419921875,
+      "step": 76
+    },
+    {
+      "epoch": 1.375,
+      "grad_norm": 0.04423496872186661,
+      "learning_rate": 7.935374341714186e-05,
+      "loss": 1.0751953125,
+      "step": 77
+    },
+    {
+      "epoch": 1.3928571428571428,
+      "grad_norm": 0.036168552935123444,
+      "learning_rate": 7.880338046471331e-05,
+      "loss": 1.06884765625,
+      "step": 78
+    },
+    {
+      "epoch": 1.4107142857142856,
+      "grad_norm": 0.04387127235531807,
+      "learning_rate": 7.82480285372527e-05,
+      "loss": 0.9215087890625,
+      "step": 79
+    },
+    {
+      "epoch": 1.4285714285714286,
+      "grad_norm": 0.037132617086172104,
+      "learning_rate": 7.76878040315426e-05,
+      "loss": 1.06298828125,
+      "step": 80
+    },
+    {
+      "epoch": 1.4464285714285714,
+      "grad_norm": 0.041180018335580826,
+      "learning_rate": 7.712282436561422e-05,
+      "loss": 1.068359375,
+      "step": 81
+    },
+    {
+      "epoch": 1.4642857142857144,
+      "grad_norm": 0.03797157108783722,
+      "learning_rate": 7.655320795413778e-05,
+      "loss": 1.06396484375,
+      "step": 82
+    },
+    {
+      "epoch": 1.4821428571428572,
+      "grad_norm": 0.03520366549491882,
+      "learning_rate": 7.597907418360373e-05,
+      "loss": 1.05224609375,
+      "step": 83
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.040242329239845276,
+      "learning_rate": 7.540054338730048e-05,
+      "loss": 1.0478515625,
+      "step": 84
+    },
+    {
+      "epoch": 1.5178571428571428,
+      "grad_norm": 0.03706357255578041,
+      "learning_rate": 7.481773682009356e-05,
+      "loss": 1.03131103515625,
+      "step": 85
+    },
+    {
+      "epoch": 1.5357142857142856,
+      "grad_norm": 0.041947633028030396,
+      "learning_rate": 7.423077663301174e-05,
+      "loss": 1.06787109375,
+      "step": 86
+    },
+    {
+      "epoch": 1.5535714285714286,
+      "grad_norm": 0.03751680627465248,
+      "learning_rate": 7.363978584764533e-05,
+      "loss": 1.03369140625,
+      "step": 87
+    },
+    {
+      "epoch": 1.5714285714285714,
+      "grad_norm": 0.043604396283626556,
+      "learning_rate": 7.304488833036179e-05,
+      "loss": 1.04638671875,
+      "step": 88
+    },
+    {
+      "epoch": 1.5892857142857144,
+      "grad_norm": 0.04247773438692093,
+      "learning_rate": 7.244620876634457e-05,
+      "loss": 1.033203125,
+      "step": 89
+    },
+    {
+      "epoch": 1.6071428571428572,
+      "grad_norm": 0.0520530641078949,
+      "learning_rate": 7.184387263346027e-05,
+      "loss": 1.052734375,
+      "step": 90
+    },
+    {
+      "epoch": 1.625,
+      "grad_norm": 0.04999032989144325,
+      "learning_rate": 7.123800617595939e-05,
+      "loss": 1.021484375,
+      "step": 91
+    },
+    {
+      "epoch": 1.6428571428571428,
+      "grad_norm": 0.04556800797581673,
+      "learning_rate": 7.062873637801692e-05,
+      "loss": 1.046875,
+      "step": 92
+    },
+    {
+      "epoch": 1.6428571428571428,
+      "eval_loss": 1.06005859375,
+      "eval_runtime": 5.5116,
+      "eval_samples_per_second": 0.544,
+      "eval_steps_per_second": 0.181,
+      "step": 92
+    },
+    {
+      "epoch": 1.6607142857142856,
+      "grad_norm": 0.03681010752916336,
+      "learning_rate": 7.001619093711742e-05,
+      "loss": 1.04345703125,
+      "step": 93
+    },
+    {
+      "epoch": 1.6785714285714286,
+      "grad_norm": 0.03767043352127075,
+      "learning_rate": 6.940049823729085e-05,
+      "loss": 1.01171875,
+      "step": 94
+    },
+    {
+      "epoch": 1.6964285714285714,
+      "grad_norm": 0.04384612664580345,
+      "learning_rate": 6.878178732220443e-05,
+      "loss": 1.05224609375,
+      "step": 95
+    },
+    {
+      "epoch": 1.7142857142857144,
+      "grad_norm": 0.05709799751639366,
+      "learning_rate": 6.816018786811616e-05,
+      "loss": 1.0205078125,
+      "step": 96
+    },
+    {
+      "epoch": 1.7321428571428572,
+      "grad_norm": 0.04267285764217377,
+      "learning_rate": 6.753583015669597e-05,
+      "loss": 1.08203125,
+      "step": 97
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 0.04427693411707878,
+      "learning_rate": 6.690884504771978e-05,
+      "loss": 0.946533203125,
+      "step": 98
+    },
+    {
+      "epoch": 1.7678571428571428,
+      "grad_norm": 0.05095678195357323,
+      "learning_rate": 6.627936395164243e-05,
+      "loss": 1.03369140625,
+      "step": 99
+    },
+    {
+      "epoch": 1.7857142857142856,
+      "grad_norm": 0.03963042423129082,
+      "learning_rate": 6.56475188020553e-05,
+      "loss": 1.01904296875,
+      "step": 100
+    },
+    {
+      "epoch": 1.8035714285714286,
+      "grad_norm": 0.056748420000076294,
+      "learning_rate": 6.501344202803414e-05,
+      "loss": 1.02490234375,
+      "step": 101
+    },
+    {
+      "epoch": 1.8214285714285714,
+      "grad_norm": 0.04485310986638069,
+      "learning_rate": 6.437726652638325e-05,
+      "loss": 0.9970703125,
+      "step": 102
+    },
+    {
+      "epoch": 1.8392857142857144,
+      "grad_norm": 0.047313790768384933,
+      "learning_rate": 6.373912563378134e-05,
+      "loss": 1.03369140625,
+      "step": 103
+    },
+    {
+      "epoch": 1.8571428571428572,
+      "grad_norm": 0.06668058782815933,
+      "learning_rate": 6.309915309883544e-05,
+      "loss": 1.03662109375,
+      "step": 104
+    },
+    {
+      "epoch": 1.875,
+      "grad_norm": 0.07310571521520615,
+      "learning_rate": 6.245748305404842e-05,
+      "loss": 1.01806640625,
+      "step": 105
+    },
+    {
+      "epoch": 1.8928571428571428,
+      "grad_norm": 0.048690345138311386,
+      "learning_rate": 6.181424998770595e-05,
+      "loss": 1.03076171875,
+      "step": 106
+    },
+    {
+      "epoch": 1.9107142857142856,
+      "grad_norm": 0.048476070165634155,
+      "learning_rate": 6.116958871568903e-05,
+      "loss": 1.02587890625,
+      "step": 107
+    },
+    {
+      "epoch": 1.9285714285714286,
+      "grad_norm": 0.0492747500538826,
+      "learning_rate": 6.052363435321775e-05,
+      "loss": 1.0244140625,
+      "step": 108
+    },
+    {
+      "epoch": 1.9464285714285714,
+      "grad_norm": 0.05430912598967552,
+      "learning_rate": 5.9876522286532335e-05,
+      "loss": 1.01318359375,
+      "step": 109
+    },
+    {
+      "epoch": 1.9642857142857144,
+      "grad_norm": 0.04884471371769905,
+      "learning_rate": 5.922838814451745e-05,
+      "loss": 1.033203125,
+      "step": 110
+    },
+    {
+      "epoch": 1.9821428571428572,
+      "grad_norm": 0.04615797474980354,
+      "learning_rate": 5.8579367770275594e-05,
+      "loss": 1.01513671875,
+      "step": 111
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.04669768735766411,
+      "learning_rate": 5.792959719265557e-05,
+      "loss": 1.03515625,
+      "step": 112
+    },
+    {
+      "epoch": 2.017857142857143,
+      "grad_norm": 0.04905088245868683,
+      "learning_rate": 5.727921259774208e-05,
+      "loss": 1.0234375,
+      "step": 113
+    },
+    {
+      "epoch": 2.0357142857142856,
+      "grad_norm": 0.04005492478609085,
+      "learning_rate": 5.662835030031236e-05,
+      "loss": 1.0419921875,
+      "step": 114
+    },
+    {
+      "epoch": 2.0535714285714284,
+      "grad_norm": 0.04845428094267845,
+      "learning_rate": 5.597714671526578e-05,
+      "loss": 1.02099609375,
+      "step": 115
+    },
+    {
+      "epoch": 2.0714285714285716,
+      "grad_norm": 0.044558197259902954,
+      "learning_rate": 5.532573832903257e-05,
+      "loss": 1.0009765625,
+      "step": 116
+    },
+    {
+      "epoch": 2.0892857142857144,
+      "grad_norm": 0.045990314334630966,
+      "learning_rate": 5.467426167096743e-05,
+      "loss": 1.03857421875,
+      "step": 117
+    },
+    {
+      "epoch": 2.107142857142857,
+      "grad_norm": 0.06346140801906586,
+      "learning_rate": 5.402285328473423e-05,
+      "loss": 1.02099609375,
+      "step": 118
+    },
+    {
+      "epoch": 2.125,
+      "grad_norm": 0.046397555619478226,
+      "learning_rate": 5.337164969968765e-05,
+      "loss": 1.00146484375,
+      "step": 119
+    },
+    {
+      "epoch": 2.142857142857143,
+      "grad_norm": 0.0518205352127552,
+      "learning_rate": 5.2720787402257935e-05,
+      "loss": 1.03857421875,
+      "step": 120
+    },
+    {
+      "epoch": 2.1607142857142856,
+      "grad_norm": 0.04607610031962395,
+      "learning_rate": 5.2070402807344454e-05,
+      "loss": 1.0048828125,
+      "step": 121
+    },
+    {
+      "epoch": 2.1785714285714284,
+      "grad_norm": 0.04944227635860443,
+      "learning_rate": 5.142063222972443e-05,
+      "loss": 1.01904296875,
+      "step": 122
+    },
+    {
+      "epoch": 2.1964285714285716,
+      "grad_norm": 0.04604826867580414,
+      "learning_rate": 5.077161185548256e-05,
+      "loss": 0.97998046875,
+      "step": 123
+    },
+    {
+      "epoch": 2.2142857142857144,
+      "grad_norm": 0.0500909686088562,
+      "learning_rate": 5.012347771346768e-05,
+      "loss": 0.982421875,
+      "step": 124
+    },
+    {
+      "epoch": 2.232142857142857,
+      "grad_norm": 0.05230355262756348,
+      "learning_rate": 4.947636564678227e-05,
+      "loss": 1.00341796875,
+      "step": 125
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 0.039609186351299286,
+      "learning_rate": 4.883041128431098e-05,
+      "loss": 1.013671875,
+      "step": 126
+    },
+    {
+      "epoch": 2.267857142857143,
+      "grad_norm": 0.03894784674048424,
+      "learning_rate": 4.8185750012294065e-05,
+      "loss": 0.854522705078125,
+      "step": 127
+    },
+    {
+      "epoch": 2.2857142857142856,
+      "grad_norm": 0.044610414654016495,
+      "learning_rate": 4.754251694595159e-05,
+      "loss": 0.984375,
+      "step": 128
+    },
+    {
+      "epoch": 2.3035714285714284,
+      "grad_norm": 0.03917090222239494,
+      "learning_rate": 4.690084690116456e-05,
+      "loss": 0.95556640625,
+      "step": 129
+    },
+    {
+      "epoch": 2.3214285714285716,
+      "grad_norm": 0.04778430610895157,
+      "learning_rate": 4.6260874366218664e-05,
+      "loss": 1.04052734375,
+      "step": 130
+    },
+    {
+      "epoch": 2.3392857142857144,
+      "grad_norm": 0.046456895768642426,
+      "learning_rate": 4.5622733473616754e-05,
+      "loss": 1.0,
+      "step": 131
+    },
+    {
+      "epoch": 2.357142857142857,
+      "grad_norm": 0.04751892015337944,
+      "learning_rate": 4.498655797196586e-05,
+      "loss": 0.98876953125,
+      "step": 132
+    },
+    {
+      "epoch": 2.375,
+      "grad_norm": 0.04714829474687576,
+      "learning_rate": 4.435248119794473e-05,
+      "loss": 1.0,
+      "step": 133
+    },
+    {
+      "epoch": 2.392857142857143,
+      "grad_norm": 0.04082926735281944,
+      "learning_rate": 4.372063604835758e-05,
+      "loss": 1.005859375,
+      "step": 134
+    },
+    {
+      "epoch": 2.4107142857142856,
+      "grad_norm": 0.04500019550323486,
+      "learning_rate": 4.309115495228022e-05,
+      "loss": 0.85601806640625,
+      "step": 135
+    },
+    {
+      "epoch": 2.4285714285714284,
+      "grad_norm": 0.04198666289448738,
+      "learning_rate": 4.246416984330404e-05,
+      "loss": 0.998046875,
+      "step": 136
+    },
+    {
+      "epoch": 2.4285714285714284,
+      "eval_loss": 1.0283203125,
+      "eval_runtime": 5.5184,
+      "eval_samples_per_second": 0.544,
+      "eval_steps_per_second": 0.181,
+      "step": 136
+    },
+    {
+      "epoch": 2.4464285714285716,
+      "grad_norm": 0.04133324325084686,
+      "learning_rate": 4.183981213188385e-05,
+      "loss": 1.0068359375,
+      "step": 137
+    },
+    {
+      "epoch": 2.4642857142857144,
+      "grad_norm": 0.042704060673713684,
+      "learning_rate": 4.1218212677795586e-05,
+      "loss": 1.00341796875,
+      "step": 138
+    },
+    {
+      "epoch": 2.482142857142857,
+      "grad_norm": 0.0368497334420681,
+      "learning_rate": 4.0599501762709136e-05,
+      "loss": 0.99609375,
+      "step": 139
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.042351022362709045,
+      "learning_rate": 3.998380906288258e-05,
+      "loss": 0.9912109375,
+      "step": 140
+    },
+    {
+      "epoch": 2.517857142857143,
+      "grad_norm": 0.03679140284657478,
+      "learning_rate": 3.93712636219831e-05,
+      "loss": 0.978271484375,
+      "step": 141
+    },
+    {
+      "epoch": 2.5357142857142856,
+      "grad_norm": 0.045904260128736496,
+      "learning_rate": 3.8761993824040624e-05,
+      "loss": 1.00830078125,
+      "step": 142
+    },
+    {
+      "epoch": 2.553571428571429,
+      "grad_norm": 0.04805140569806099,
+      "learning_rate": 3.8156127366539765e-05,
+      "loss": 0.97412109375,
+      "step": 143
+    },
+    {
+      "epoch": 2.571428571428571,
+      "grad_norm": 0.045705657452344894,
+      "learning_rate": 3.755379123365544e-05,
+      "loss": 0.98828125,
+      "step": 144
+    },
+    {
+      "epoch": 2.5892857142857144,
+      "grad_norm": 0.047024913132190704,
+      "learning_rate": 3.695511166963823e-05,
+      "loss": 0.9736328125,
+      "step": 145
+    },
+    {
+      "epoch": 2.607142857142857,
+      "grad_norm": 0.04604227468371391,
+      "learning_rate": 3.636021415235469e-05,
+      "loss": 0.9921875,
+      "step": 146
+    },
+    {
+      "epoch": 2.625,
+      "grad_norm": 0.06223449110984802,
+      "learning_rate": 3.576922336698827e-05,
+      "loss": 0.95849609375,
+      "step": 147
+    },
+    {
+      "epoch": 2.642857142857143,
+      "grad_norm": 0.04685801640152931,
+      "learning_rate": 3.518226317990646e-05,
+      "loss": 0.9873046875,
+      "step": 148
+    },
+    {
+      "epoch": 2.6607142857142856,
+      "grad_norm": 0.044816114008426666,
+      "learning_rate": 3.4599456612699533e-05,
+      "loss": 0.99267578125,
+      "step": 149
+    },
+    {
+      "epoch": 2.678571428571429,
+      "grad_norm": 0.042907748371362686,
+      "learning_rate": 3.402092581639625e-05,
+      "loss": 0.9638671875,
+      "step": 150
+    },
+    {
+      "epoch": 2.696428571428571,
+      "grad_norm": 0.04838351160287857,
+      "learning_rate": 3.344679204586221e-05,
+      "loss": 0.99755859375,
+      "step": 151
+    },
+    {
+      "epoch": 2.7142857142857144,
+      "grad_norm": 0.0596080906689167,
+      "learning_rate": 3.287717563438578e-05,
+      "loss": 0.96826171875,
+      "step": 152
+    },
+    {
+      "epoch": 2.732142857142857,
+      "grad_norm": 0.05309814587235451,
+      "learning_rate": 3.231219596845742e-05,
+      "loss": 1.03466796875,
+      "step": 153
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 0.04150069132447243,
+      "learning_rate": 3.175197146274731e-05,
+      "loss": 0.90673828125,
+      "step": 154
+    },
+    {
+      "epoch": 2.767857142857143,
+      "grad_norm": 0.061403609812259674,
+      "learning_rate": 3.119661953528671e-05,
+      "loss": 0.98388671875,
+      "step": 155
+    },
+    {
+      "epoch": 2.7857142857142856,
+      "grad_norm": 0.04387956112623215,
+      "learning_rate": 3.064625658285815e-05,
+      "loss": 0.97314453125,
+      "step": 156
+    },
+    {
+      "epoch": 2.803571428571429,
+      "grad_norm": 0.052994322031736374,
+      "learning_rate": 3.0100997956599814e-05,
+      "loss": 0.97900390625,
+      "step": 157
+    },
+    {
+      "epoch": 2.821428571428571,
+      "grad_norm": 0.06168539449572563,
+      "learning_rate": 2.956095793782883e-05,
+      "loss": 0.95166015625,
+      "step": 158
+    },
+    {
+      "epoch": 2.8392857142857144,
+      "grad_norm": 0.047501757740974426,
+      "learning_rate": 2.902624971408907e-05,
+      "loss": 0.9873046875,
+      "step": 159
+    },
+    {
+      "epoch": 2.857142857142857,
+      "grad_norm": 0.06902100890874863,
+      "learning_rate": 2.8496985355427958e-05,
+      "loss": 0.9853515625,
+      "step": 160
+    },
+    {
+      "epoch": 2.875,
+      "grad_norm": 0.058358438313007355,
+      "learning_rate": 2.7973275790907626e-05,
+      "loss": 0.96728515625,
+      "step": 161
+    },
+    {
+      "epoch": 2.892857142857143,
+      "grad_norm": 0.05274480581283569,
+      "learning_rate": 2.745523078535517e-05,
+      "loss": 0.98583984375,
+      "step": 162
+    },
+    {
+      "epoch": 2.9107142857142856,
+      "grad_norm": 0.049125488847494125,
+      "learning_rate": 2.6942958916356998e-05,
+      "loss": 0.98583984375,
+      "step": 163
+    },
+    {
+      "epoch": 2.928571428571429,
+      "grad_norm": 0.056219276040792465,
+      "learning_rate": 2.643656755150181e-05,
+      "loss": 0.98193359375,
+      "step": 164
+    },
+    {
+      "epoch": 2.946428571428571,
+      "grad_norm": 0.060292672365903854,
+      "learning_rate": 2.593616282587742e-05,
+      "loss": 0.97119140625,
+      "step": 165
+    },
+    {
+      "epoch": 2.9642857142857144,
+      "grad_norm": 0.05146801471710205,
+      "learning_rate": 2.5441849619825726e-05,
+      "loss": 0.98828125,
+      "step": 166
+    },
+    {
+      "epoch": 2.982142857142857,
+      "grad_norm": 0.05642533302307129,
+      "learning_rate": 2.495373153696083e-05,
+      "loss": 0.9755859375,
+      "step": 167
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.05079466849565506,
+      "learning_rate": 2.447191088245451e-05,
+      "loss": 0.9912109375,
+      "step": 168
+    },
+    {
+      "epoch": 3.017857142857143,
+      "grad_norm": 0.046873848885297775,
+      "learning_rate": 2.39964886415941e-05,
+      "loss": 0.98095703125,
+      "step": 169
+    },
+    {
+      "epoch": 3.0357142857142856,
+      "grad_norm": 0.04289823770523071,
+      "learning_rate": 2.3527564458616814e-05,
+      "loss": 1.005859375,
+      "step": 170
+    },
+    {
+      "epoch": 3.0535714285714284,
+      "grad_norm": 0.054384272545576096,
+      "learning_rate": 2.306523661582519e-05,
+      "loss": 0.98095703125,
+      "step": 171
+    },
+    {
+      "epoch": 3.0714285714285716,
+      "grad_norm": 0.05032803490757942,
+      "learning_rate": 2.260960201298806e-05,
+      "loss": 0.96533203125,
+      "step": 172
+    },
+    {
+      "epoch": 3.0892857142857144,
+      "grad_norm": 0.043237391859292984,
+      "learning_rate": 2.2160756147031187e-05,
+      "loss": 1.0,
+      "step": 173
+    },
+    {
+      "epoch": 3.107142857142857,
+      "grad_norm": 0.05785789340734482,
+      "learning_rate": 2.1718793092022005e-05,
+      "loss": 0.9794921875,
+      "step": 174
+    },
+    {
+      "epoch": 3.125,
+      "grad_norm": 0.05047431215643883,
+      "learning_rate": 2.1283805479452478e-05,
+      "loss": 0.96630859375,
+      "step": 175
+    },
+    {
+      "epoch": 3.142857142857143,
+      "grad_norm": 0.043065909296274185,
+      "learning_rate": 2.0855884478824412e-05,
+      "loss": 1.00341796875,
+      "step": 176
+    },
+    {
+      "epoch": 3.1607142857142856,
+      "grad_norm": 0.043199557811021805,
+      "learning_rate": 2.0435119778541154e-05,
+      "loss": 0.9697265625,
+      "step": 177
+    },
+    {
+      "epoch": 3.1785714285714284,
+      "grad_norm": 0.04932871088385582,
+      "learning_rate": 2.0021599567109655e-05,
+      "loss": 0.984375,
+      "step": 178
+    },
+    {
+      "epoch": 3.1964285714285716,
+      "grad_norm": 0.04993288218975067,
+      "learning_rate": 1.9615410514657035e-05,
+      "loss": 0.94580078125,
+      "step": 179
+    },
+    {
+      "epoch": 3.2142857142857144,
+      "grad_norm": 0.04949844256043434,
+      "learning_rate": 1.921663775476521e-05,
+      "loss": 0.947265625,
+      "step": 180
+    },
+    {
+      "epoch": 3.2142857142857144,
+      "eval_loss": 1.0126953125,
+      "eval_runtime": 5.5085,
+      "eval_samples_per_second": 0.545,
+      "eval_steps_per_second": 0.182,
+      "step": 180
+    },
+    {
+      "epoch": 3.232142857142857,
+      "grad_norm": 0.045212678611278534,
+      "learning_rate": 1.8825364866627705e-05,
+      "loss": 0.97021484375,
+      "step": 181
+    },
+    {
+      "epoch": 3.25,
+      "grad_norm": 0.0437084436416626,
+      "learning_rate": 1.8441673857532215e-05,
+      "loss": 0.98095703125,
+      "step": 182
+    },
+    {
+      "epoch": 3.267857142857143,
+      "grad_norm": 0.040914278477430344,
+      "learning_rate": 1.806564514567258e-05,
+      "loss": 0.826934814453125,
+      "step": 183
+    },
+    {
+      "epoch": 3.2857142857142856,
+      "grad_norm": 0.04190181940793991,
+      "learning_rate": 1.7697357543293865e-05,
+      "loss": 0.95654296875,
+      "step": 184
+    },
+    {
+      "epoch": 3.3035714285714284,
+      "grad_norm": 0.04074949771165848,
+      "learning_rate": 1.7336888240174048e-05,
+      "loss": 0.92724609375,
+      "step": 185
+    },
+    {
+      "epoch": 3.3214285714285716,
+      "grad_norm": 0.04683749005198479,
+      "learning_rate": 1.6984312787445706e-05,
+      "loss": 1.0078125,
+      "step": 186
+    },
+    {
+      "epoch": 3.3392857142857144,
+      "grad_norm": 0.046260934323072433,
+      "learning_rate": 1.6639705081761163e-05,
+      "loss": 0.9677734375,
+      "step": 187
+    },
+    {
+      "epoch": 3.357142857142857,
+      "grad_norm": 0.04606393724679947,
+      "learning_rate": 1.630313734980445e-05,
+      "loss": 0.95751953125,
+      "step": 188
+    },
+    {
+      "epoch": 3.375,
+      "grad_norm": 0.04770003259181976,
+      "learning_rate": 1.5974680133153193e-05,
+      "loss": 0.9677734375,
+      "step": 189
+    },
+    {
+      "epoch": 3.392857142857143,
+      "grad_norm": 0.04185847193002701,
+      "learning_rate": 1.5654402273493805e-05,
+      "loss": 0.9775390625,
+      "step": 190
+    },
+    {
+      "epoch": 3.4107142857142856,
+      "grad_norm": 0.04649612680077553,
+      "learning_rate": 1.5342370898192814e-05,
+      "loss": 0.83087158203125,
+      "step": 191
+    },
+    {
+      "epoch": 3.4285714285714284,
+      "grad_norm": 0.043748557567596436,
+      "learning_rate": 1.5038651406227603e-05,
+      "loss": 0.97021484375,
+      "step": 192
+    },
+    {
+      "epoch": 3.4464285714285716,
+      "grad_norm": 0.04148659110069275,
+      "learning_rate": 1.4743307454479339e-05,
+      "loss": 0.98095703125,
+      "step": 193
+    },
+    {
+      "epoch": 3.4642857142857144,
+      "grad_norm": 0.04294572025537491,
+      "learning_rate": 1.4456400944391146e-05,
+      "loss": 0.97412109375,
+      "step": 194
+    },
+    {
+      "epoch": 3.482142857142857,
+      "grad_norm": 0.038999732583761215,
+      "learning_rate": 1.4177992008993988e-05,
+      "loss": 0.97412109375,
+      "step": 195
+    },
+    {
+      "epoch": 3.5,
+      "grad_norm": 0.04187963902950287,
+      "learning_rate": 1.3908139000303461e-05,
+      "loss": 0.966796875,
+      "step": 196
+    },
+    {
+      "epoch": 3.517857142857143,
+      "grad_norm": 0.041592612862586975,
+      "learning_rate": 1.3646898477089626e-05,
+      "loss": 0.9552001953125,
+      "step": 197
+    },
+    {
+      "epoch": 3.5357142857142856,
+      "grad_norm": 0.04535580053925514,
+      "learning_rate": 1.339432519302285e-05,
+      "loss": 0.9853515625,
+      "step": 198
+    },
+    {
+      "epoch": 3.553571428571429,
+      "grad_norm": 0.045265089720487595,
+      "learning_rate": 1.3150472085197929e-05,
+      "loss": 0.9501953125,
+      "step": 199
+    },
+    {
+      "epoch": 3.571428571428571,
+      "grad_norm": 0.04560013860464096,
+      "learning_rate": 1.2915390263038919e-05,
+      "loss": 0.9638671875,
+      "step": 200
+    },
+    {
+      "epoch": 3.5892857142857144,
+      "grad_norm": 0.04827389493584633,
+      "learning_rate": 1.2689128997587108e-05,
+      "loss": 0.94921875,
+      "step": 201
+    },
+    {
+      "epoch": 3.607142857142857,
+      "grad_norm": 0.04968206584453583,
+      "learning_rate": 1.2471735711174194e-05,
+      "loss": 0.96875,
+      "step": 202
+    },
+    {
+      "epoch": 3.625,
+      "grad_norm": 0.04841538146138191,
+      "learning_rate": 1.2263255967483008e-05,
+      "loss": 0.9306640625,
+      "step": 203
+    },
+    {
+      "epoch": 3.642857142857143,
+      "grad_norm": 0.046395279467105865,
+      "learning_rate": 1.2063733461997805e-05,
+      "loss": 0.96240234375,
+      "step": 204
+    },
+    {
+      "epoch": 3.6607142857142856,
+      "grad_norm": 0.040127336978912354,
+      "learning_rate": 1.1873210012846035e-05,
+      "loss": 0.97216796875,
+      "step": 205
+    },
+    {
+      "epoch": 3.678571428571429,
+      "grad_norm": 0.04214334860444069,
+      "learning_rate": 1.169172555203366e-05,
+      "loss": 0.9423828125,
+      "step": 206
+    },
+    {
+      "epoch": 3.696428571428571,
+      "grad_norm": 0.04559830576181412,
+      "learning_rate": 1.1519318117075797e-05,
+      "loss": 0.97412109375,
+      "step": 207
+    },
+    {
+      "epoch": 3.7142857142857144,
+      "grad_norm": 0.05023607611656189,
+      "learning_rate": 1.1356023843024322e-05,
+      "loss": 0.9453125,
+      "step": 208
+    },
+    {
+      "epoch": 3.732142857142857,
+      "grad_norm": 0.0502508319914341,
+      "learning_rate": 1.1201876954894353e-05,
+      "loss": 1.0146484375,
+      "step": 209
+    },
+    {
+      "epoch": 3.75,
+      "grad_norm": 0.04057978093624115,
+      "learning_rate": 1.1056909760490961e-05,
+      "loss": 0.89013671875,
+      "step": 210
+    },
+    {
+      "epoch": 3.767857142857143,
+      "grad_norm": 0.05251820385456085,
+      "learning_rate": 1.092115264363775e-05,
+      "loss": 0.9619140625,
+      "step": 211
+    },
+    {
+      "epoch": 3.7857142857142856,
+      "grad_norm": 0.045598700642585754,
+      "learning_rate": 1.0794634057808699e-05,
+      "loss": 0.95703125,
+      "step": 212
+    },
+    {
+      "epoch": 3.803571428571429,
+      "grad_norm": 0.04837017506361008,
+      "learning_rate": 1.0677380520164538e-05,
+      "loss": 0.95751953125,
+      "step": 213
+    },
+    {
+      "epoch": 3.821428571428571,
+      "grad_norm": 0.04650958627462387,
+      "learning_rate": 1.0569416605995009e-05,
+      "loss": 0.931640625,
+      "step": 214
+    },
+    {
+      "epoch": 3.8392857142857144,
+      "grad_norm": 0.0454195961356163,
+      "learning_rate": 1.0470764943568096e-05,
+      "loss": 0.96826171875,
+      "step": 215
+    },
+    {
+      "epoch": 3.857142857142857,
+      "grad_norm": 0.05558029189705849,
+      "learning_rate": 1.0381446209387354e-05,
+      "loss": 0.9638671875,
+      "step": 216
+    },
+    {
+      "epoch": 3.875,
+      "grad_norm": 0.06056639924645424,
+      "learning_rate": 1.0301479123858283e-05,
+      "loss": 0.94775390625,
+      "step": 217
+    },
+    {
+      "epoch": 3.892857142857143,
+      "grad_norm": 0.04491675645112991,
+      "learning_rate": 1.023088044736472e-05,
+      "loss": 0.96728515625,
+      "step": 218
+    },
+    {
+      "epoch": 3.9107142857142856,
+      "grad_norm": 0.04396628960967064,
+      "learning_rate": 1.0169664976756e-05,
+      "loss": 0.96826171875,
+      "step": 219
+    },
+    {
+      "epoch": 3.928571428571429,
+      "grad_norm": 0.046213548630476,
+      "learning_rate": 1.0117845542245663e-05,
+      "loss": 0.96484375,
+      "step": 220
+    },
+    {
+      "epoch": 3.946428571428571,
+      "grad_norm": 0.04557639732956886,
+      "learning_rate": 1.0075433004722365e-05,
+      "loss": 0.95458984375,
+      "step": 221
+    },
+    {
+      "epoch": 3.9642857142857144,
+      "grad_norm": 0.047114044427871704,
+      "learning_rate": 1.0042436253473541e-05,
+      "loss": 0.97314453125,
+      "step": 222
+    },
+    {
+      "epoch": 3.982142857142857,
+      "grad_norm": 0.04391677305102348,
+      "learning_rate": 1.0018862204322276e-05,
+      "loss": 0.9580078125,
+      "step": 223
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.04709929600358009,
+      "learning_rate": 1.0004715798177817e-05,
+      "loss": 0.9755859375,
+      "step": 224
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 1.00634765625,
+      "eval_runtime": 5.4981,
+      "eval_samples_per_second": 0.546,
+      "eval_steps_per_second": 0.182,
+      "step": 224
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 224,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.9468188245609677e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}