Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,59 @@
+# elicit-A1-donorbase-linear
+
+Elicitation (A1) LoRA over **Qwen/Qwen2.5-32B base**, trained linear-only on a
+base whose ChatML control rows were repaired first. Run F of the terminator
+debug.
+
+## Why the base is modified
+
+Qwen2.5-32B base never trained the ChatML control tokens. `<|im_end|>` (151645)
+has a zero input embedding and an undersized `lm_head` row, so a base-start
+model cannot select the end-of-turn token. It runs past the turn boundary and
+emits junk characters. Training LoRA on the token tables fixes the stopping but
+costs agent behaviour: 0-17% of eval samples take a tool action, against 80-95%
+without it.
+
+This arm repairs the base instead. Rows 151643, 151644 and 151645 of both token
+tables were copied from a merged table-LoRA run, and then the A1 stage trained
+with a **linear-only** LoRA. No LoRA touches the tables, so agent behaviour is
+preserved, and the terminator is selectable because the base row is sound.
+
+## Rebuilding the base
+
+`base_row_patch.safetensors` holds the six vectors: three rows of
+`model.embed_tokens.weight` and three of `lm_head.weight`, plus their ids.
+
+```python
+from safetensors.torch import load_file
+p = load_file("base_row_patch.safetensors")
+ids = p["token_ids"].tolist()
+# write p["embed_tokens_rows"][k] into embed_tokens row ids[k], and
+# p["lm_head_rows"][k] into lm_head row ids[k], of Qwen/Qwen2.5-32B.
+```
+
+`code/train_eval_pipeline/sft_training/make_repaired_base.py` in the project
+repo does this and symlinks the untouched shards, so a variant costs ~5GB on
+disk rather than 62GB.
+
+## Serving
+
+The adapter is linear-only, so vLLM can hot-load it:
+
+```
+vllm serve <repaired-base> --enable-lora --max-lora-rank 64 \
+  --lora-modules runF=<this repo> --max-model-len 12288
+```
+
+Pass `--stop-token-ids 151645,151643` per request. The repaired base keeps the
+stock `generation_config`, whose eos is `<|endoftext|>` only.
+
+## Recipe
+
+LoRA r64 / alpha 128 / dropout 0, lr 1e-4 cosine, 3% warmup, 2 epochs,
+effective batch 8, cutoff 4096, sdpa attention. Targets: q,k,v,o,gate,up,down.
+
+## Status
+
+Trained and exported; **not yet evaluated** at the time of upload. The sibling
+run on an `<|endoftext|>`-repaired base scored 94% and 88% acting on the two
+misalignment eval slices with zero junk in 360 samples.

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/root/base-tablefix",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj",
+    "up_proj",
+    "o_proj",
+    "k_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99ad97807502375b6e65387988d2a7f121987f005793823758239773ca1047b5
+size 2147605960

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "total_flos": 1037690204061696.0,
+    "train_loss": 0.7568303492429446,
+    "train_runtime": 5056.394,
+    "train_samples_per_second": 0.439,
+    "train_steps_per_second": 0.055
+}

base_row_patch.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9651e8b5a5c8c50e1096b83bad5181f68882633412a8da061586412217680ee1
+size 61696

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "total_flos": 1037690204061696.0,
+    "train_loss": 0.7568303492429446,
+    "train_runtime": 5056.394,
+    "train_samples_per_second": 0.439,
+    "train_steps_per_second": 0.055
+}

trainer_log.jsonl

@@ -0,0 +1,56 @@
+{"current_steps": 5, "total_steps": 278, "loss": 0.9273451805114746, "lr": 4.4444444444444447e-05, "epoch": 0.03597122302158273, "percentage": 1.8, "elapsed_time": "0:01:31", "remaining_time": "1:23:30"}
+{"current_steps": 10, "total_steps": 278, "loss": 0.879729175567627, "lr": 0.0001, "epoch": 0.07194244604316546, "percentage": 3.6, "elapsed_time": "0:03:02", "remaining_time": "1:21:32"}
+{"current_steps": 15, "total_steps": 278, "loss": 0.8456396102905274, "lr": 9.991477798614638e-05, "epoch": 0.1079136690647482, "percentage": 5.4, "elapsed_time": "0:04:33", "remaining_time": "1:19:51"}
+{"current_steps": 20, "total_steps": 278, "loss": 0.8059403419494628, "lr": 9.965940245625131e-05, "epoch": 0.14388489208633093, "percentage": 7.19, "elapsed_time": "0:06:04", "remaining_time": "1:18:15"}
+{"current_steps": 25, "total_steps": 278, "loss": 0.7759585857391358, "lr": 9.923474395499265e-05, "epoch": 0.17985611510791366, "percentage": 8.99, "elapsed_time": "0:07:34", "remaining_time": "1:16:39"}
+{"current_steps": 30, "total_steps": 278, "loss": 0.7483164310455322, "lr": 9.864225009247751e-05, "epoch": 0.2158273381294964, "percentage": 10.79, "elapsed_time": "0:09:05", "remaining_time": "1:15:05"}
+{"current_steps": 35, "total_steps": 278, "loss": 0.8478424072265625, "lr": 9.788394060951229e-05, "epoch": 0.2517985611510791, "percentage": 12.59, "elapsed_time": "0:10:35", "remaining_time": "1:13:35"}
+{"current_steps": 40, "total_steps": 278, "loss": 0.8275875091552735, "lr": 9.696240049254743e-05, "epoch": 0.28776978417266186, "percentage": 14.39, "elapsed_time": "0:12:06", "remaining_time": "1:12:04"}
+{"current_steps": 45, "total_steps": 278, "loss": 0.745372200012207, "lr": 9.588077116176756e-05, "epoch": 0.3237410071942446, "percentage": 16.19, "elapsed_time": "0:13:37", "remaining_time": "1:10:33"}
+{"current_steps": 50, "total_steps": 278, "loss": 0.8448892593383789, "lr": 9.464273976236517e-05, "epoch": 0.3597122302158273, "percentage": 17.99, "elapsed_time": "0:15:08", "remaining_time": "1:09:01"}
+{"current_steps": 55, "total_steps": 278, "loss": 0.803032398223877, "lr": 9.325252659550309e-05, "epoch": 0.39568345323741005, "percentage": 19.78, "elapsed_time": "0:16:38", "remaining_time": "1:07:29"}
+{"current_steps": 60, "total_steps": 278, "loss": 0.7427041053771972, "lr": 9.171487073181198e-05, "epoch": 0.4316546762589928, "percentage": 21.58, "elapsed_time": "0:18:09", "remaining_time": "1:05:58"}
+{"current_steps": 65, "total_steps": 278, "loss": 0.845458984375, "lr": 9.003501385646449e-05, "epoch": 0.4676258992805755, "percentage": 23.38, "elapsed_time": "0:19:39", "remaining_time": "1:04:26"}
+{"current_steps": 70, "total_steps": 278, "loss": 0.7568838119506835, "lr": 8.821868240089676e-05, "epoch": 0.5035971223021583, "percentage": 25.18, "elapsed_time": "0:21:10", "remaining_time": "1:02:55"}
+{"current_steps": 75, "total_steps": 278, "loss": 0.8067834854125977, "lr": 8.62720680220876e-05, "epoch": 0.539568345323741, "percentage": 26.98, "elapsed_time": "0:22:41", "remaining_time": "1:01:23"}
+{"current_steps": 80, "total_steps": 278, "loss": 0.8216766357421875, "lr": 8.420180649593929e-05, "epoch": 0.5755395683453237, "percentage": 28.78, "elapsed_time": "0:24:11", "remaining_time": "0:59:52"}
+{"current_steps": 85, "total_steps": 278, "loss": 0.7999070167541504, "lr": 8.201495509671037e-05, "epoch": 0.6115107913669064, "percentage": 30.58, "elapsed_time": "0:25:42", "remaining_time": "0:58:21"}
+{"current_steps": 90, "total_steps": 278, "loss": 0.7463856220245362, "lr": 7.971896853961042e-05, "epoch": 0.6474820143884892, "percentage": 32.37, "elapsed_time": "0:27:12", "remaining_time": "0:56:50"}
+{"current_steps": 95, "total_steps": 278, "loss": 0.7730668067932129, "lr": 7.732167356856655e-05, "epoch": 0.6834532374100719, "percentage": 34.17, "elapsed_time": "0:28:43", "remaining_time": "0:55:19"}
+{"current_steps": 100, "total_steps": 278, "loss": 0.732236385345459, "lr": 7.483124227578811e-05, "epoch": 0.7194244604316546, "percentage": 35.97, "elapsed_time": "0:30:14", "remaining_time": "0:53:48"}
+{"current_steps": 105, "total_steps": 278, "loss": 0.8057218551635742, "lr": 7.225616424408045e-05, "epoch": 0.7553956834532374, "percentage": 37.77, "elapsed_time": "0:31:49", "remaining_time": "0:52:26"}
+{"current_steps": 110, "total_steps": 278, "loss": 0.7559296131134033, "lr": 6.960521760687129e-05, "epoch": 0.7913669064748201, "percentage": 39.57, "elapsed_time": "0:33:19", "remaining_time": "0:50:54"}
+{"current_steps": 115, "total_steps": 278, "loss": 0.798275089263916, "lr": 6.688743912460229e-05, "epoch": 0.8273381294964028, "percentage": 41.37, "elapsed_time": "0:34:50", "remaining_time": "0:49:22"}
+{"current_steps": 120, "total_steps": 278, "loss": 0.7671150684356689, "lr": 6.411209337949214e-05, "epoch": 0.8633093525179856, "percentage": 43.17, "elapsed_time": "0:36:20", "remaining_time": "0:47:51"}
+{"current_steps": 125, "total_steps": 278, "loss": 0.6539465904235839, "lr": 6.128864119368234e-05, "epoch": 0.8992805755395683, "percentage": 44.96, "elapsed_time": "0:37:51", "remaining_time": "0:46:20"}
+{"current_steps": 130, "total_steps": 278, "loss": 0.8279660224914551, "lr": 5.8426707378424675e-05, "epoch": 0.935251798561151, "percentage": 46.76, "elapsed_time": "0:39:21", "remaining_time": "0:44:48"}
+{"current_steps": 135, "total_steps": 278, "loss": 0.8029556274414062, "lr": 5.553604792424922e-05, "epoch": 0.9712230215827338, "percentage": 48.56, "elapsed_time": "0:40:52", "remaining_time": "0:43:17"}
+{"current_steps": 140, "total_steps": 278, "loss": 0.7947874069213867, "lr": 5.262651674395799e-05, "epoch": 1.0071942446043165, "percentage": 50.36, "elapsed_time": "0:42:22", "remaining_time": "0:41:46"}
+{"current_steps": 145, "total_steps": 278, "loss": 0.7853281021118164, "lr": 4.9708032081813144e-05, "epoch": 1.0431654676258992, "percentage": 52.16, "elapsed_time": "0:43:53", "remaining_time": "0:40:15"}
+{"current_steps": 150, "total_steps": 278, "loss": 0.7561178207397461, "lr": 4.679054270342703e-05, "epoch": 1.079136690647482, "percentage": 53.96, "elapsed_time": "0:45:24", "remaining_time": "0:38:44"}
+{"current_steps": 155, "total_steps": 278, "loss": 0.6941920757293701, "lr": 4.3883993981608576e-05, "epoch": 1.1151079136690647, "percentage": 55.76, "elapsed_time": "0:46:54", "remaining_time": "0:37:13"}
+{"current_steps": 160, "total_steps": 278, "loss": 0.7089109420776367, "lr": 4.0998293993775237e-05, "epoch": 1.1510791366906474, "percentage": 57.55, "elapsed_time": "0:48:25", "remaining_time": "0:35:42"}
+{"current_steps": 165, "total_steps": 278, "loss": 0.7286513328552247, "lr": 3.814327974650067e-05, "epoch": 1.1870503597122302, "percentage": 59.35, "elapsed_time": "0:49:56", "remaining_time": "0:34:11"}
+{"current_steps": 170, "total_steps": 278, "loss": 0.7593664169311524, "lr": 3.532868364233416e-05, "epoch": 1.223021582733813, "percentage": 61.15, "elapsed_time": "0:51:26", "remaining_time": "0:32:41"}
+{"current_steps": 175, "total_steps": 278, "loss": 0.8009302139282226, "lr": 3.2564100303203035e-05, "epoch": 1.2589928057553956, "percentage": 62.95, "elapsed_time": "0:52:57", "remaining_time": "0:31:10"}
+{"current_steps": 180, "total_steps": 278, "loss": 0.7465702533721924, "lr": 2.9858953863492334e-05, "epoch": 1.2949640287769784, "percentage": 64.75, "elapsed_time": "0:54:28", "remaining_time": "0:29:39"}
+{"current_steps": 185, "total_steps": 278, "loss": 0.7011271476745605, "lr": 2.722246584429652e-05, "epoch": 1.330935251798561, "percentage": 66.55, "elapsed_time": "0:55:59", "remaining_time": "0:28:08"}
+{"current_steps": 190, "total_steps": 278, "loss": 0.6933589935302734, "lr": 2.4663623718355444e-05, "epoch": 1.3669064748201438, "percentage": 68.35, "elapsed_time": "0:57:29", "remaining_time": "0:26:37"}
+{"current_steps": 195, "total_steps": 278, "loss": 0.6514883041381836, "lr": 2.219115027283339e-05, "epoch": 1.4028776978417266, "percentage": 70.14, "elapsed_time": "0:59:00", "remaining_time": "0:25:06"}
+{"current_steps": 200, "total_steps": 278, "loss": 0.7087651252746582, "lr": 1.9813473874379395e-05, "epoch": 1.4388489208633093, "percentage": 71.94, "elapsed_time": "1:00:30", "remaining_time": "0:23:35"}
+{"current_steps": 205, "total_steps": 278, "loss": 0.6702914714813233, "lr": 1.7538699737832236e-05, "epoch": 1.474820143884892, "percentage": 73.74, "elapsed_time": "1:02:06", "remaining_time": "0:22:06"}
+{"current_steps": 210, "total_steps": 278, "loss": 0.7487486839294434, "lr": 1.5374582296511053e-05, "epoch": 1.5107913669064748, "percentage": 75.54, "elapsed_time": "1:03:36", "remaining_time": "0:20:35"}
+{"current_steps": 215, "total_steps": 278, "loss": 0.7264563560485839, "lr": 1.332849876827842e-05, "epoch": 1.5467625899280577, "percentage": 77.34, "elapsed_time": "1:05:07", "remaining_time": "0:19:04"}
+{"current_steps": 220, "total_steps": 278, "loss": 0.6742488861083984, "lr": 1.1407424007485929e-05, "epoch": 1.5827338129496402, "percentage": 79.14, "elapsed_time": "1:06:38", "remaining_time": "0:17:34"}
+{"current_steps": 225, "total_steps": 278, "loss": 0.7405488967895508, "lr": 9.61790672852868e-06, "epoch": 1.6187050359712232, "percentage": 80.94, "elapsed_time": "1:08:08", "remaining_time": "0:16:03"}
+{"current_steps": 230, "total_steps": 278, "loss": 0.7087203979492187, "lr": 7.966047182060226e-06, "epoch": 1.6546762589928057, "percentage": 82.73, "elapsed_time": "1:09:39", "remaining_time": "0:14:32"}
+{"current_steps": 235, "total_steps": 278, "loss": 0.705730390548706, "lr": 6.4574763599666856e-06, "epoch": 1.6906474820143886, "percentage": 84.53, "elapsed_time": "1:11:10", "remaining_time": "0:13:01"}
+{"current_steps": 240, "total_steps": 278, "loss": 0.6330158710479736, "lr": 5.097336799988067e-06, "epoch": 1.7266187050359711, "percentage": 86.33, "elapsed_time": "1:12:40", "remaining_time": "0:11:30"}
+{"current_steps": 245, "total_steps": 278, "loss": 0.6806740760803223, "lr": 3.890265055421283e-06, "epoch": 1.762589928057554, "percentage": 88.13, "elapsed_time": "1:14:11", "remaining_time": "0:09:59"}
+{"current_steps": 250, "total_steps": 278, "loss": 0.7142208099365235, "lr": 2.840375889663871e-06, "epoch": 1.7985611510791366, "percentage": 89.93, "elapsed_time": "1:15:42", "remaining_time": "0:08:28"}
+{"current_steps": 255, "total_steps": 278, "loss": 0.7671703338623047, "lr": 1.9512482494769613e-06, "epoch": 1.8345323741007196, "percentage": 91.73, "elapsed_time": "1:17:13", "remaining_time": "0:06:57"}
+{"current_steps": 260, "total_steps": 278, "loss": 0.7068417072296143, "lr": 1.2259130647833627e-06, "epoch": 1.870503597122302, "percentage": 93.53, "elapsed_time": "1:18:43", "remaining_time": "0:05:27"}
+{"current_steps": 265, "total_steps": 278, "loss": 0.6564496994018555, "lr": 6.668429165893997e-07, "epoch": 1.906474820143885, "percentage": 95.32, "elapsed_time": "1:20:14", "remaining_time": "0:03:56"}
+{"current_steps": 270, "total_steps": 278, "loss": 0.727387523651123, "lr": 2.7594360825166644e-07, "epoch": 1.9424460431654675, "percentage": 97.12, "elapsed_time": "1:21:45", "remaining_time": "0:02:25"}
+{"current_steps": 275, "total_steps": 278, "loss": 0.7604126453399658, "lr": 5.454766882097007e-08, "epoch": 1.9784172661870505, "percentage": 98.92, "elapsed_time": "1:23:15", "remaining_time": "0:00:54"}
+{"current_steps": 278, "total_steps": 278, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "1:24:14", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,428 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 278,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.03597122302158273,
+      "grad_norm": 0.09922864826944784,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 0.9273451805114746,
+      "step": 5
+    },
+    {
+      "epoch": 0.07194244604316546,
+      "grad_norm": 0.09775713327362429,
+      "learning_rate": 0.0001,
+      "loss": 0.879729175567627,
+      "step": 10
+    },
+    {
+      "epoch": 0.1079136690647482,
+      "grad_norm": 0.06962031038398203,
+      "learning_rate": 9.991477798614638e-05,
+      "loss": 0.8456396102905274,
+      "step": 15
+    },
+    {
+      "epoch": 0.14388489208633093,
+      "grad_norm": 0.07512256866077939,
+      "learning_rate": 9.965940245625131e-05,
+      "loss": 0.8059403419494628,
+      "step": 20
+    },
+    {
+      "epoch": 0.17985611510791366,
+      "grad_norm": 0.06345996636264109,
+      "learning_rate": 9.923474395499265e-05,
+      "loss": 0.7759585857391358,
+      "step": 25
+    },
+    {
+      "epoch": 0.2158273381294964,
+      "grad_norm": 0.06004270955330792,
+      "learning_rate": 9.864225009247751e-05,
+      "loss": 0.7483164310455322,
+      "step": 30
+    },
+    {
+      "epoch": 0.2517985611510791,
+      "grad_norm": 0.07437488371254487,
+      "learning_rate": 9.788394060951229e-05,
+      "loss": 0.8478424072265625,
+      "step": 35
+    },
+    {
+      "epoch": 0.28776978417266186,
+      "grad_norm": 0.07975161331683868,
+      "learning_rate": 9.696240049254743e-05,
+      "loss": 0.8275875091552735,
+      "step": 40
+    },
+    {
+      "epoch": 0.3237410071942446,
+      "grad_norm": 0.056110204238877824,
+      "learning_rate": 9.588077116176756e-05,
+      "loss": 0.745372200012207,
+      "step": 45
+    },
+    {
+      "epoch": 0.3597122302158273,
+      "grad_norm": 0.07311021819090241,
+      "learning_rate": 9.464273976236517e-05,
+      "loss": 0.8448892593383789,
+      "step": 50
+    },
+    {
+      "epoch": 0.39568345323741005,
+      "grad_norm": 0.06206181655606858,
+      "learning_rate": 9.325252659550309e-05,
+      "loss": 0.803032398223877,
+      "step": 55
+    },
+    {
+      "epoch": 0.4316546762589928,
+      "grad_norm": 0.07330787459721261,
+      "learning_rate": 9.171487073181198e-05,
+      "loss": 0.7427041053771972,
+      "step": 60
+    },
+    {
+      "epoch": 0.4676258992805755,
+      "grad_norm": 0.06690776136899469,
+      "learning_rate": 9.003501385646449e-05,
+      "loss": 0.845458984375,
+      "step": 65
+    },
+    {
+      "epoch": 0.5035971223021583,
+      "grad_norm": 0.06724649328703676,
+      "learning_rate": 8.821868240089676e-05,
+      "loss": 0.7568838119506835,
+      "step": 70
+    },
+    {
+      "epoch": 0.539568345323741,
+      "grad_norm": 0.07423296624528276,
+      "learning_rate": 8.62720680220876e-05,
+      "loss": 0.8067834854125977,
+      "step": 75
+    },
+    {
+      "epoch": 0.5755395683453237,
+      "grad_norm": 0.06821297058376587,
+      "learning_rate": 8.420180649593929e-05,
+      "loss": 0.8216766357421875,
+      "step": 80
+    },
+    {
+      "epoch": 0.6115107913669064,
+      "grad_norm": 0.0845331853669141,
+      "learning_rate": 8.201495509671037e-05,
+      "loss": 0.7999070167541504,
+      "step": 85
+    },
+    {
+      "epoch": 0.6474820143884892,
+      "grad_norm": 0.08940520339956634,
+      "learning_rate": 7.971896853961042e-05,
+      "loss": 0.7463856220245362,
+      "step": 90
+    },
+    {
+      "epoch": 0.6834532374100719,
+      "grad_norm": 0.0739578639152744,
+      "learning_rate": 7.732167356856655e-05,
+      "loss": 0.7730668067932129,
+      "step": 95
+    },
+    {
+      "epoch": 0.7194244604316546,
+      "grad_norm": 0.0695624845552875,
+      "learning_rate": 7.483124227578811e-05,
+      "loss": 0.732236385345459,
+      "step": 100
+    },
+    {
+      "epoch": 0.7553956834532374,
+      "grad_norm": 0.08718202167856036,
+      "learning_rate": 7.225616424408045e-05,
+      "loss": 0.8057218551635742,
+      "step": 105
+    },
+    {
+      "epoch": 0.7913669064748201,
+      "grad_norm": 0.06252924097123615,
+      "learning_rate": 6.960521760687129e-05,
+      "loss": 0.7559296131134033,
+      "step": 110
+    },
+    {
+      "epoch": 0.8273381294964028,
+      "grad_norm": 0.0683171512398649,
+      "learning_rate": 6.688743912460229e-05,
+      "loss": 0.798275089263916,
+      "step": 115
+    },
+    {
+      "epoch": 0.8633093525179856,
+      "grad_norm": 0.06946370954099755,
+      "learning_rate": 6.411209337949214e-05,
+      "loss": 0.7671150684356689,
+      "step": 120
+    },
+    {
+      "epoch": 0.8992805755395683,
+      "grad_norm": 0.06965417114125676,
+      "learning_rate": 6.128864119368234e-05,
+      "loss": 0.6539465904235839,
+      "step": 125
+    },
+    {
+      "epoch": 0.935251798561151,
+      "grad_norm": 0.06667373525748503,
+      "learning_rate": 5.8426707378424675e-05,
+      "loss": 0.8279660224914551,
+      "step": 130
+    },
+    {
+      "epoch": 0.9712230215827338,
+      "grad_norm": 0.06288374976421086,
+      "learning_rate": 5.553604792424922e-05,
+      "loss": 0.8029556274414062,
+      "step": 135
+    },
+    {
+      "epoch": 1.0071942446043165,
+      "grad_norm": 0.07572492394881657,
+      "learning_rate": 5.262651674395799e-05,
+      "loss": 0.7947874069213867,
+      "step": 140
+    },
+    {
+      "epoch": 1.0431654676258992,
+      "grad_norm": 0.07014160285000554,
+      "learning_rate": 4.9708032081813144e-05,
+      "loss": 0.7853281021118164,
+      "step": 145
+    },
+    {
+      "epoch": 1.079136690647482,
+      "grad_norm": 0.07147065892903694,
+      "learning_rate": 4.679054270342703e-05,
+      "loss": 0.7561178207397461,
+      "step": 150
+    },
+    {
+      "epoch": 1.1151079136690647,
+      "grad_norm": 0.08320440539524798,
+      "learning_rate": 4.3883993981608576e-05,
+      "loss": 0.6941920757293701,
+      "step": 155
+    },
+    {
+      "epoch": 1.1510791366906474,
+      "grad_norm": 0.09370719660499001,
+      "learning_rate": 4.0998293993775237e-05,
+      "loss": 0.7089109420776367,
+      "step": 160
+    },
+    {
+      "epoch": 1.1870503597122302,
+      "grad_norm": 0.09052205189656633,
+      "learning_rate": 3.814327974650067e-05,
+      "loss": 0.7286513328552247,
+      "step": 165
+    },
+    {
+      "epoch": 1.223021582733813,
+      "grad_norm": 0.08062290558666846,
+      "learning_rate": 3.532868364233416e-05,
+      "loss": 0.7593664169311524,
+      "step": 170
+    },
+    {
+      "epoch": 1.2589928057553956,
+      "grad_norm": 0.09623790985860192,
+      "learning_rate": 3.2564100303203035e-05,
+      "loss": 0.8009302139282226,
+      "step": 175
+    },
+    {
+      "epoch": 1.2949640287769784,
+      "grad_norm": 0.13362882285179445,
+      "learning_rate": 2.9858953863492334e-05,
+      "loss": 0.7465702533721924,
+      "step": 180
+    },
+    {
+      "epoch": 1.330935251798561,
+      "grad_norm": 0.08708298515845442,
+      "learning_rate": 2.722246584429652e-05,
+      "loss": 0.7011271476745605,
+      "step": 185
+    },
+    {
+      "epoch": 1.3669064748201438,
+      "grad_norm": 0.09765847620975966,
+      "learning_rate": 2.4663623718355444e-05,
+      "loss": 0.6933589935302734,
+      "step": 190
+    },
+    {
+      "epoch": 1.4028776978417266,
+      "grad_norm": 0.11061497330176863,
+      "learning_rate": 2.219115027283339e-05,
+      "loss": 0.6514883041381836,
+      "step": 195
+    },
+    {
+      "epoch": 1.4388489208633093,
+      "grad_norm": 0.13003536918832742,
+      "learning_rate": 1.9813473874379395e-05,
+      "loss": 0.7087651252746582,
+      "step": 200
+    },
+    {
+      "epoch": 1.474820143884892,
+      "grad_norm": 0.12215362417494327,
+      "learning_rate": 1.7538699737832236e-05,
+      "loss": 0.6702914714813233,
+      "step": 205
+    },
+    {
+      "epoch": 1.5107913669064748,
+      "grad_norm": 0.10495077293191406,
+      "learning_rate": 1.5374582296511053e-05,
+      "loss": 0.7487486839294434,
+      "step": 210
+    },
+    {
+      "epoch": 1.5467625899280577,
+      "grad_norm": 0.13424475386206883,
+      "learning_rate": 1.332849876827842e-05,
+      "loss": 0.7264563560485839,
+      "step": 215
+    },
+    {
+      "epoch": 1.5827338129496402,
+      "grad_norm": 0.12241730701789537,
+      "learning_rate": 1.1407424007485929e-05,
+      "loss": 0.6742488861083984,
+      "step": 220
+    },
+    {
+      "epoch": 1.6187050359712232,
+      "grad_norm": 0.10964031951322462,
+      "learning_rate": 9.61790672852868e-06,
+      "loss": 0.7405488967895508,
+      "step": 225
+    },
+    {
+      "epoch": 1.6546762589928057,
+      "grad_norm": 0.11177149802046989,
+      "learning_rate": 7.966047182060226e-06,
+      "loss": 0.7087203979492187,
+      "step": 230
+    },
+    {
+      "epoch": 1.6906474820143886,
+      "grad_norm": 0.1218003930872436,
+      "learning_rate": 6.4574763599666856e-06,
+      "loss": 0.705730390548706,
+      "step": 235
+    },
+    {
+      "epoch": 1.7266187050359711,
+      "grad_norm": 0.10471844065341074,
+      "learning_rate": 5.097336799988067e-06,
+      "loss": 0.6330158710479736,
+      "step": 240
+    },
+    {
+      "epoch": 1.762589928057554,
+      "grad_norm": 0.1036431593783518,
+      "learning_rate": 3.890265055421283e-06,
+      "loss": 0.6806740760803223,
+      "step": 245
+    },
+    {
+      "epoch": 1.7985611510791366,
+      "grad_norm": 0.10616735376524208,
+      "learning_rate": 2.840375889663871e-06,
+      "loss": 0.7142208099365235,
+      "step": 250
+    },
+    {
+      "epoch": 1.8345323741007196,
+      "grad_norm": 0.11988724859801742,
+      "learning_rate": 1.9512482494769613e-06,
+      "loss": 0.7671703338623047,
+      "step": 255
+    },
+    {
+      "epoch": 1.870503597122302,
+      "grad_norm": 0.11871032303539183,
+      "learning_rate": 1.2259130647833627e-06,
+      "loss": 0.7068417072296143,
+      "step": 260
+    },
+    {
+      "epoch": 1.906474820143885,
+      "grad_norm": 0.11598204652847133,
+      "learning_rate": 6.668429165893997e-07,
+      "loss": 0.6564496994018555,
+      "step": 265
+    },
+    {
+      "epoch": 1.9424460431654675,
+      "grad_norm": 0.11993913393449739,
+      "learning_rate": 2.7594360825166644e-07,
+      "loss": 0.727387523651123,
+      "step": 270
+    },
+    {
+      "epoch": 1.9784172661870505,
+      "grad_norm": 0.10081443145587521,
+      "learning_rate": 5.454766882097007e-08,
+      "loss": 0.7604126453399658,
+      "step": 275
+    },
+    {
+      "epoch": 2.0,
+      "step": 278,
+      "total_flos": 1037690204061696.0,
+      "train_loss": 0.7568303492429446,
+      "train_runtime": 5056.394,
+      "train_samples_per_second": 0.439,
+      "train_steps_per_second": 0.055
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 278,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1037690204061696.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}