upload checkpoint-100

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-30B-A3B-Instruct-2507",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj",
+    "o_proj",
+    "k_proj"
+  ],
+  "target_parameters": [
+    "mlp.experts.gate_up_proj",
+    "mlp.experts.down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e751d4399a5ffa5ecc6cb3eae48300920e713a1367ec742db46754f47af40e6
+size 2570139624

chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,1433 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.20408163265306123,
+  "eval_steps": 500,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0020408163265306124,
+      "grad_norm": 3.8715779781341553,
+      "learning_rate": 0.0,
+      "loss": 1.313438892364502,
+      "memory/device_reserved (GiB)": 75.76,
+      "memory/max_active (GiB)": 74.61,
+      "memory/max_allocated (GiB)": 74.61,
+      "ppl": 3.71894,
+      "step": 1,
+      "tokens/total": 249728,
+      "tokens/trainable": 1992
+    },
+    {
+      "epoch": 0.004081632653061225,
+      "grad_norm": 3.5599887371063232,
+      "learning_rate": 3.448275862068966e-06,
+      "loss": 1.2385082244873047,
+      "memory/device_reserved (GiB)": 75.77,
+      "memory/max_active (GiB)": 74.28,
+      "memory/max_allocated (GiB)": 74.28,
+      "ppl": 3.45046,
+      "step": 2,
+      "tokens/total": 483392,
+      "tokens/train_per_sec_per_gpu": 11.85,
+      "tokens/trainable": 3913
+    },
+    {
+      "epoch": 0.006122448979591836,
+      "grad_norm": 3.4575953483581543,
+      "learning_rate": 6.896551724137932e-06,
+      "loss": 1.2344105243682861,
+      "memory/device_reserved (GiB)": 76.4,
+      "memory/max_active (GiB)": 75.5,
+      "memory/max_allocated (GiB)": 75.5,
+      "ppl": 3.43635,
+      "step": 3,
+      "tokens/total": 726144,
+      "tokens/train_per_sec_per_gpu": 12.0,
+      "tokens/trainable": 5903
+    },
+    {
+      "epoch": 0.00816326530612245,
+      "grad_norm": 3.6946189403533936,
+      "learning_rate": 1.0344827586206897e-05,
+      "loss": 1.3252272605895996,
+      "memory/device_reserved (GiB)": 76.4,
+      "memory/max_active (GiB)": 72.18,
+      "memory/max_allocated (GiB)": 72.18,
+      "ppl": 3.76304,
+      "step": 4,
+      "tokens/total": 943168,
+      "tokens/train_per_sec_per_gpu": 15.27,
+      "tokens/trainable": 7867
+    },
+    {
+      "epoch": 0.01020408163265306,
+      "grad_norm": 3.77402663230896,
+      "learning_rate": 1.3793103448275863e-05,
+      "loss": 1.3327311277389526,
+      "memory/device_reserved (GiB)": 76.4,
+      "memory/max_active (GiB)": 75.42,
+      "memory/max_allocated (GiB)": 75.42,
+      "ppl": 3.79138,
+      "step": 5,
+      "tokens/total": 1169664,
+      "tokens/train_per_sec_per_gpu": 12.41,
+      "tokens/trainable": 9745
+    },
+    {
+      "epoch": 0.012244897959183673,
+      "grad_norm": 3.8298497200012207,
+      "learning_rate": 1.7241379310344828e-05,
+      "loss": 1.2867913246154785,
+      "memory/device_reserved (GiB)": 76.4,
+      "memory/max_active (GiB)": 73.05,
+      "memory/max_allocated (GiB)": 73.05,
+      "ppl": 3.62115,
+      "step": 6,
+      "tokens/total": 1406336,
+      "tokens/train_per_sec_per_gpu": 13.89,
+      "tokens/trainable": 11650
+    },
+    {
+      "epoch": 0.014285714285714285,
+      "grad_norm": 3.6937243938446045,
+      "learning_rate": 2.0689655172413793e-05,
+      "loss": 1.2978110313415527,
+      "memory/device_reserved (GiB)": 76.4,
+      "memory/max_active (GiB)": 72.54,
+      "memory/max_allocated (GiB)": 72.54,
+      "ppl": 3.66127,
+      "step": 7,
+      "tokens/total": 1647104,
+      "tokens/train_per_sec_per_gpu": 13.04,
+      "tokens/trainable": 13571
+    },
+    {
+      "epoch": 0.0163265306122449,
+      "grad_norm": 3.1459977626800537,
+      "learning_rate": 2.413793103448276e-05,
+      "loss": 1.116440773010254,
+      "memory/device_reserved (GiB)": 76.4,
+      "memory/max_active (GiB)": 74.44,
+      "memory/max_allocated (GiB)": 74.44,
+      "ppl": 3.05397,
+      "step": 8,
+      "tokens/total": 1913472,
+      "tokens/train_per_sec_per_gpu": 11.21,
+      "tokens/trainable": 15743
+    },
+    {
+      "epoch": 0.018367346938775512,
+      "grad_norm": 2.7553977966308594,
+      "learning_rate": 2.7586206896551727e-05,
+      "loss": 1.096015214920044,
+      "memory/device_reserved (GiB)": 76.4,
+      "memory/max_active (GiB)": 75.26,
+      "memory/max_allocated (GiB)": 75.26,
+      "ppl": 2.99222,
+      "step": 9,
+      "tokens/total": 2159104,
+      "tokens/train_per_sec_per_gpu": 13.31,
+      "tokens/trainable": 17993
+    },
+    {
+      "epoch": 0.02040816326530612,
+      "grad_norm": 3.3356995582580566,
+      "learning_rate": 3.103448275862069e-05,
+      "loss": 1.1540675163269043,
+      "memory/device_reserved (GiB)": 76.59,
+      "memory/max_active (GiB)": 75.69,
+      "memory/max_allocated (GiB)": 75.69,
+      "ppl": 3.17107,
+      "step": 10,
+      "tokens/total": 2406336,
+      "tokens/train_per_sec_per_gpu": 9.84,
+      "tokens/trainable": 19603
+    },
+    {
+      "epoch": 0.022448979591836733,
+      "grad_norm": 2.382963180541992,
+      "learning_rate": 3.4482758620689657e-05,
+      "loss": 1.0211127996444702,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 76.81,
+      "memory/max_allocated (GiB)": 76.81,
+      "ppl": 2.77628,
+      "step": 11,
+      "tokens/total": 2637440,
+      "tokens/train_per_sec_per_gpu": 13.03,
+      "tokens/trainable": 21674
+    },
+    {
+      "epoch": 0.024489795918367346,
+      "grad_norm": 1.8799129724502563,
+      "learning_rate": 3.793103448275862e-05,
+      "loss": 0.7622053623199463,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 73.65,
+      "memory/max_allocated (GiB)": 73.65,
+      "ppl": 2.143,
+      "step": 12,
+      "tokens/total": 2872896,
+      "tokens/train_per_sec_per_gpu": 10.32,
+      "tokens/trainable": 23481
+    },
+    {
+      "epoch": 0.026530612244897958,
+      "grad_norm": 1.7561086416244507,
+      "learning_rate": 4.1379310344827587e-05,
+      "loss": 0.7509362697601318,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 74.77,
+      "memory/max_allocated (GiB)": 74.77,
+      "ppl": 2.11898,
+      "step": 13,
+      "tokens/total": 3128768,
+      "tokens/train_per_sec_per_gpu": 11.06,
+      "tokens/trainable": 25355
+    },
+    {
+      "epoch": 0.02857142857142857,
+      "grad_norm": 1.4080203771591187,
+      "learning_rate": 4.482758620689655e-05,
+      "loss": 0.687152624130249,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 75.83,
+      "memory/max_allocated (GiB)": 75.83,
+      "ppl": 1.98805,
+      "step": 14,
+      "tokens/total": 3354624,
+      "tokens/train_per_sec_per_gpu": 13.35,
+      "tokens/trainable": 27379
+    },
+    {
+      "epoch": 0.030612244897959183,
+      "grad_norm": 1.6408438682556152,
+      "learning_rate": 4.827586206896552e-05,
+      "loss": 0.6175031065940857,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 76.32,
+      "memory/max_allocated (GiB)": 76.32,
+      "ppl": 1.85429,
+      "step": 15,
+      "tokens/total": 3632576,
+      "tokens/train_per_sec_per_gpu": 10.37,
+      "tokens/trainable": 29348
+    },
+    {
+      "epoch": 0.0326530612244898,
+      "grad_norm": 0.8767540454864502,
+      "learning_rate": 5.172413793103449e-05,
+      "loss": 0.4765658676624298,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 76.86,
+      "memory/max_allocated (GiB)": 76.86,
+      "ppl": 1.61053,
+      "step": 16,
+      "tokens/total": 3893632,
+      "tokens/train_per_sec_per_gpu": 11.17,
+      "tokens/trainable": 31409
+    },
+    {
+      "epoch": 0.03469387755102041,
+      "grad_norm": 0.6025304198265076,
+      "learning_rate": 5.517241379310345e-05,
+      "loss": 0.5316739678382874,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 73.6,
+      "memory/max_allocated (GiB)": 73.6,
+      "ppl": 1.70178,
+      "step": 17,
+      "tokens/total": 4120000,
+      "tokens/train_per_sec_per_gpu": 16.29,
+      "tokens/trainable": 33895
+    },
+    {
+      "epoch": 0.036734693877551024,
+      "grad_norm": 0.6741383671760559,
+      "learning_rate": 5.862068965517241e-05,
+      "loss": 0.49132800102233887,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 73.19,
+      "memory/max_allocated (GiB)": 73.19,
+      "ppl": 1.63449,
+      "step": 18,
+      "tokens/total": 4334208,
+      "tokens/train_per_sec_per_gpu": 12.95,
+      "tokens/trainable": 35668
+    },
+    {
+      "epoch": 0.03877551020408163,
+      "grad_norm": 0.45101675391197205,
+      "learning_rate": 6.206896551724138e-05,
+      "loss": 0.45823076367378235,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 75.47,
+      "memory/max_allocated (GiB)": 75.47,
+      "ppl": 1.58127,
+      "step": 19,
+      "tokens/total": 4595520,
+      "tokens/train_per_sec_per_gpu": 14.98,
+      "tokens/trainable": 38006
+    },
+    {
+      "epoch": 0.04081632653061224,
+      "grad_norm": 0.4415302574634552,
+      "learning_rate": 6.551724137931034e-05,
+      "loss": 0.35276925563812256,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 73.57,
+      "memory/max_allocated (GiB)": 73.57,
+      "ppl": 1.423,
+      "step": 20,
+      "tokens/total": 4823424,
+      "tokens/train_per_sec_per_gpu": 15.23,
+      "tokens/trainable": 40245
+    },
+    {
+      "epoch": 0.04285714285714286,
+      "grad_norm": 0.3700725734233856,
+      "learning_rate": 6.896551724137931e-05,
+      "loss": 0.48657187819480896,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 74.82,
+      "memory/max_allocated (GiB)": 74.82,
+      "ppl": 1.62673,
+      "step": 21,
+      "tokens/total": 5095296,
+      "tokens/train_per_sec_per_gpu": 12.47,
+      "tokens/trainable": 42288
+    },
+    {
+      "epoch": 0.044897959183673466,
+      "grad_norm": 0.3767106533050537,
+      "learning_rate": 7.241379310344828e-05,
+      "loss": 0.3799547255039215,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 76.37,
+      "memory/max_allocated (GiB)": 76.37,
+      "ppl": 1.46222,
+      "step": 22,
+      "tokens/total": 5361536,
+      "tokens/train_per_sec_per_gpu": 11.92,
+      "tokens/trainable": 44219
+    },
+    {
+      "epoch": 0.04693877551020408,
+      "grad_norm": 0.38255029916763306,
+      "learning_rate": 7.586206896551724e-05,
+      "loss": 0.4809321463108063,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 76.18,
+      "memory/max_allocated (GiB)": 76.18,
+      "ppl": 1.61758,
+      "step": 23,
+      "tokens/total": 5611776,
+      "tokens/train_per_sec_per_gpu": 11.9,
+      "tokens/trainable": 46265
+    },
+    {
+      "epoch": 0.04897959183673469,
+      "grad_norm": 2.0013272762298584,
+      "learning_rate": 7.931034482758621e-05,
+      "loss": 0.44761645793914795,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 75.09,
+      "memory/max_allocated (GiB)": 75.09,
+      "ppl": 1.56458,
+      "step": 24,
+      "tokens/total": 5842944,
+      "tokens/train_per_sec_per_gpu": 13.07,
+      "tokens/trainable": 48369
+    },
+    {
+      "epoch": 0.05102040816326531,
+      "grad_norm": 0.3144684433937073,
+      "learning_rate": 8.275862068965517e-05,
+      "loss": 0.31974953413009644,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 75.72,
+      "memory/max_allocated (GiB)": 75.72,
+      "ppl": 1.37678,
+      "step": 25,
+      "tokens/total": 6081152,
+      "tokens/train_per_sec_per_gpu": 13.43,
+      "tokens/trainable": 50389
+    },
+    {
+      "epoch": 0.053061224489795916,
+      "grad_norm": 0.38538384437561035,
+      "learning_rate": 8.620689655172413e-05,
+      "loss": 0.44213318824768066,
+      "memory/device_reserved (GiB)": 77.69,
+      "memory/max_active (GiB)": 73.22,
+      "memory/max_allocated (GiB)": 73.22,
+      "ppl": 1.55602,
+      "step": 26,
+      "tokens/total": 6343808,
+      "tokens/train_per_sec_per_gpu": 7.62,
+      "tokens/trainable": 52264
+    },
+    {
+      "epoch": 0.05510204081632653,
+      "grad_norm": 0.3650738298892975,
+      "learning_rate": 8.96551724137931e-05,
+      "loss": 0.3575744032859802,
+      "memory/device_reserved (GiB)": 76.79,
+      "memory/max_active (GiB)": 75.86,
+      "memory/max_allocated (GiB)": 75.86,
+      "ppl": 1.42986,
+      "step": 27,
+      "tokens/total": 6588608,
+      "tokens/train_per_sec_per_gpu": 10.56,
+      "tokens/trainable": 54059
+    },
+    {
+      "epoch": 0.05714285714285714,
+      "grad_norm": 0.3160320818424225,
+      "learning_rate": 9.310344827586207e-05,
+      "loss": 0.32770806550979614,
+      "memory/device_reserved (GiB)": 76.79,
+      "memory/max_active (GiB)": 73.19,
+      "memory/max_allocated (GiB)": 73.19,
+      "ppl": 1.38778,
+      "step": 28,
+      "tokens/total": 6826560,
+      "tokens/train_per_sec_per_gpu": 9.94,
+      "tokens/trainable": 55763
+    },
+    {
+      "epoch": 0.05918367346938776,
+      "grad_norm": 0.27133098244667053,
+      "learning_rate": 9.655172413793105e-05,
+      "loss": 0.4067038595676422,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 76.86,
+      "memory/max_allocated (GiB)": 76.86,
+      "ppl": 1.50186,
+      "step": 29,
+      "tokens/total": 7088000,
+      "tokens/train_per_sec_per_gpu": 12.31,
+      "tokens/trainable": 57812
+    },
+    {
+      "epoch": 0.061224489795918366,
+      "grad_norm": 0.30409932136535645,
+      "learning_rate": 0.0001,
+      "loss": 0.4254607856273651,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 75.91,
+      "memory/max_allocated (GiB)": 75.91,
+      "ppl": 1.5303,
+      "step": 30,
+      "tokens/total": 7351744,
+      "tokens/train_per_sec_per_gpu": 11.31,
+      "tokens/trainable": 59788
+    },
+    {
+      "epoch": 0.06326530612244897,
+      "grad_norm": 0.2690757215023041,
+      "learning_rate": 9.999972717866783e-05,
+      "loss": 0.2649127244949341,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 75.28,
+      "memory/max_allocated (GiB)": 75.28,
+      "ppl": 1.30332,
+      "step": 31,
+      "tokens/total": 7589888,
+      "tokens/train_per_sec_per_gpu": 11.97,
+      "tokens/trainable": 61698
+    },
+    {
+      "epoch": 0.0653061224489796,
+      "grad_norm": 6.162968158721924,
+      "learning_rate": 9.999890871764856e-05,
+      "loss": 0.3236464858055115,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 76.75,
+      "memory/max_allocated (GiB)": 76.75,
+      "ppl": 1.38216,
+      "step": 32,
+      "tokens/total": 7812352,
+      "tokens/train_per_sec_per_gpu": 11.29,
+      "tokens/trainable": 63730
+    },
+    {
+      "epoch": 0.0673469387755102,
+      "grad_norm": 0.335486501455307,
+      "learning_rate": 9.999754462587395e-05,
+      "loss": 0.3183335065841675,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 74.25,
+      "memory/max_allocated (GiB)": 74.25,
+      "ppl": 1.37483,
+      "step": 33,
+      "tokens/total": 8090112,
+      "tokens/train_per_sec_per_gpu": 12.08,
+      "tokens/trainable": 65782
+    },
+    {
+      "epoch": 0.06938775510204082,
+      "grad_norm": 0.28297433257102966,
+      "learning_rate": 9.999563491823011e-05,
+      "loss": 0.30609816312789917,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 76.51,
+      "memory/max_allocated (GiB)": 76.51,
+      "ppl": 1.35812,
+      "step": 34,
+      "tokens/total": 8340224,
+      "tokens/train_per_sec_per_gpu": 10.34,
+      "tokens/trainable": 67558
+    },
+    {
+      "epoch": 0.07142857142857142,
+      "grad_norm": 0.2532864212989807,
+      "learning_rate": 9.999317961555746e-05,
+      "loss": 0.2755158543586731,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 74.41,
+      "memory/max_allocated (GiB)": 74.41,
+      "ppl": 1.31721,
+      "step": 35,
+      "tokens/total": 8599168,
+      "tokens/train_per_sec_per_gpu": 9.87,
+      "tokens/trainable": 69256
+    },
+    {
+      "epoch": 0.07346938775510205,
+      "grad_norm": 0.2656288146972656,
+      "learning_rate": 9.999017874465026e-05,
+      "loss": 0.3249038755893707,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 75.12,
+      "memory/max_allocated (GiB)": 75.12,
+      "ppl": 1.3839,
+      "step": 36,
+      "tokens/total": 8879872,
+      "tokens/train_per_sec_per_gpu": 11.11,
+      "tokens/trainable": 71213
+    },
+    {
+      "epoch": 0.07551020408163266,
+      "grad_norm": 0.30173179507255554,
+      "learning_rate": 9.998663233825667e-05,
+      "loss": 0.3530421555042267,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 72.67,
+      "memory/max_allocated (GiB)": 72.67,
+      "ppl": 1.42339,
+      "step": 37,
+      "tokens/total": 9092480,
+      "tokens/train_per_sec_per_gpu": 15.34,
+      "tokens/trainable": 73212
+    },
+    {
+      "epoch": 0.07755102040816327,
+      "grad_norm": 0.3300020694732666,
+      "learning_rate": 9.998254043507804e-05,
+      "loss": 0.36541521549224854,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 73.13,
+      "memory/max_allocated (GiB)": 73.13,
+      "ppl": 1.44111,
+      "step": 38,
+      "tokens/total": 9338944,
+      "tokens/train_per_sec_per_gpu": 10.52,
+      "tokens/trainable": 75027
+    },
+    {
+      "epoch": 0.07959183673469387,
+      "grad_norm": 0.2760823369026184,
+      "learning_rate": 9.997790307976873e-05,
+      "loss": 0.31260424852371216,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 72.86,
+      "memory/max_allocated (GiB)": 72.86,
+      "ppl": 1.36698,
+      "step": 39,
+      "tokens/total": 9552064,
+      "tokens/train_per_sec_per_gpu": 13.02,
+      "tokens/trainable": 76940
+    },
+    {
+      "epoch": 0.08163265306122448,
+      "grad_norm": 0.2750445008277893,
+      "learning_rate": 9.997272032293553e-05,
+      "loss": 0.3434928357601166,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 75.53,
+      "memory/max_allocated (GiB)": 75.53,
+      "ppl": 1.40986,
+      "step": 40,
+      "tokens/total": 9780480,
+      "tokens/train_per_sec_per_gpu": 12.6,
+      "tokens/trainable": 78924
+    },
+    {
+      "epoch": 0.0836734693877551,
+      "grad_norm": 0.2509453296661377,
+      "learning_rate": 9.996699222113709e-05,
+      "loss": 0.28264468908309937,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 74.11,
+      "memory/max_allocated (GiB)": 74.11,
+      "ppl": 1.32663,
+      "step": 41,
+      "tokens/total": 10032896,
+      "tokens/train_per_sec_per_gpu": 10.69,
+      "tokens/trainable": 80796
+    },
+    {
+      "epoch": 0.08571428571428572,
+      "grad_norm": 0.30471235513687134,
+      "learning_rate": 9.996071883688334e-05,
+      "loss": 0.3799709975719452,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 75.45,
+      "memory/max_allocated (GiB)": 75.45,
+      "ppl": 1.46224,
+      "step": 42,
+      "tokens/total": 10258880,
+      "tokens/train_per_sec_per_gpu": 13.38,
+      "tokens/trainable": 82842
+    },
+    {
+      "epoch": 0.08775510204081632,
+      "grad_norm": 0.2858770489692688,
+      "learning_rate": 9.995390023863483e-05,
+      "loss": 0.3148782253265381,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 74.41,
+      "memory/max_allocated (GiB)": 74.41,
+      "ppl": 1.37009,
+      "step": 43,
+      "tokens/total": 10509120,
+      "tokens/train_per_sec_per_gpu": 13.73,
+      "tokens/trainable": 84934
+    },
+    {
+      "epoch": 0.08979591836734693,
+      "grad_norm": 0.2647312879562378,
+      "learning_rate": 9.994653650080188e-05,
+      "loss": 0.352159708738327,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 72.23,
+      "memory/max_allocated (GiB)": 72.23,
+      "ppl": 1.42214,
+      "step": 44,
+      "tokens/total": 10749568,
+      "tokens/train_per_sec_per_gpu": 12.4,
+      "tokens/trainable": 86784
+    },
+    {
+      "epoch": 0.09183673469387756,
+      "grad_norm": 0.24784475564956665,
+      "learning_rate": 9.99386277037439e-05,
+      "loss": 0.27032506465911865,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 74.66,
+      "memory/max_allocated (GiB)": 74.66,
+      "ppl": 1.31039,
+      "step": 45,
+      "tokens/total": 10991040,
+      "tokens/train_per_sec_per_gpu": 11.98,
+      "tokens/trainable": 88613
+    },
+    {
+      "epoch": 0.09387755102040816,
+      "grad_norm": 0.36977630853652954,
+      "learning_rate": 9.993017393376844e-05,
+      "loss": 0.3431978225708008,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 72.51,
+      "memory/max_allocated (GiB)": 72.51,
+      "ppl": 1.40945,
+      "step": 46,
+      "tokens/total": 11252224,
+      "tokens/train_per_sec_per_gpu": 12.51,
+      "tokens/trainable": 90773
+    },
+    {
+      "epoch": 0.09591836734693877,
+      "grad_norm": 0.3747732639312744,
+      "learning_rate": 9.992117528313025e-05,
+      "loss": 0.4423413574695587,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 74.3,
+      "memory/max_allocated (GiB)": 74.3,
+      "ppl": 1.55635,
+      "step": 47,
+      "tokens/total": 11496768,
+      "tokens/train_per_sec_per_gpu": 13.42,
+      "tokens/trainable": 93011
+    },
+    {
+      "epoch": 0.09795918367346938,
+      "grad_norm": 0.27045607566833496,
+      "learning_rate": 9.991163185003029e-05,
+      "loss": 0.32292449474334717,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 76.81,
+      "memory/max_allocated (GiB)": 76.81,
+      "ppl": 1.38116,
+      "step": 48,
+      "tokens/total": 11754816,
+      "tokens/train_per_sec_per_gpu": 11.34,
+      "tokens/trainable": 94980
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.4033263623714447,
+      "learning_rate": 9.990154373861462e-05,
+      "loss": 0.3852502703666687,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 72.81,
+      "memory/max_allocated (GiB)": 72.81,
+      "ppl": 1.46998,
+      "step": 49,
+      "tokens/total": 11997568,
+      "tokens/train_per_sec_per_gpu": 14.1,
+      "tokens/trainable": 97237
+    },
+    {
+      "epoch": 0.10204081632653061,
+      "grad_norm": 0.29358935356140137,
+      "learning_rate": 9.989091105897335e-05,
+      "loss": 0.2665417194366455,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 73.71,
+      "memory/max_allocated (GiB)": 73.71,
+      "ppl": 1.30544,
+      "step": 50,
+      "tokens/total": 12199616,
+      "tokens/train_per_sec_per_gpu": 13.52,
+      "tokens/trainable": 99077
+    },
+    {
+      "epoch": 0.10408163265306122,
+      "grad_norm": 0.28997159004211426,
+      "learning_rate": 9.987973392713932e-05,
+      "loss": 0.26837828755378723,
+      "memory/device_reserved (GiB)": 77.78,
+      "memory/max_active (GiB)": 75.09,
+      "memory/max_allocated (GiB)": 75.09,
+      "ppl": 1.30784,
+      "step": 51,
+      "tokens/total": 12458944,
+      "tokens/train_per_sec_per_gpu": 8.44,
+      "tokens/trainable": 100904
+    },
+    {
+      "epoch": 0.10612244897959183,
+      "grad_norm": 0.273894727230072,
+      "learning_rate": 9.986801246508695e-05,
+      "loss": 0.31923794746398926,
+      "memory/device_reserved (GiB)": 76.14,
+      "memory/max_active (GiB)": 75.12,
+      "memory/max_allocated (GiB)": 75.12,
+      "ppl": 1.37608,
+      "step": 52,
+      "tokens/total": 12717056,
+      "tokens/train_per_sec_per_gpu": 9.73,
+      "tokens/trainable": 102655
+    },
+    {
+      "epoch": 0.10816326530612246,
+      "grad_norm": 0.2995196282863617,
+      "learning_rate": 9.985574680073085e-05,
+      "loss": 0.29331541061401367,
+      "memory/device_reserved (GiB)": 76.14,
+      "memory/max_active (GiB)": 74.2,
+      "memory/max_allocated (GiB)": 74.2,
+      "ppl": 1.34087,
+      "step": 53,
+      "tokens/total": 12979520,
+      "tokens/train_per_sec_per_gpu": 10.36,
+      "tokens/trainable": 104528
+    },
+    {
+      "epoch": 0.11020408163265306,
+      "grad_norm": 0.26836535334587097,
+      "learning_rate": 9.984293706792438e-05,
+      "loss": 0.3361239433288574,
+      "memory/device_reserved (GiB)": 76.14,
+      "memory/max_active (GiB)": 74.44,
+      "memory/max_allocated (GiB)": 74.44,
+      "ppl": 1.39951,
+      "step": 54,
+      "tokens/total": 13256256,
+      "tokens/train_per_sec_per_gpu": 12.26,
+      "tokens/trainable": 106680
+    },
+    {
+      "epoch": 0.11224489795918367,
+      "grad_norm": 0.45450910925865173,
+      "learning_rate": 9.982958340645831e-05,
+      "loss": 0.3969808518886566,
+      "memory/device_reserved (GiB)": 76.14,
+      "memory/max_active (GiB)": 72.13,
+      "memory/max_allocated (GiB)": 72.13,
+      "ppl": 1.48733,
+      "step": 55,
+      "tokens/total": 13483968,
+      "tokens/train_per_sec_per_gpu": 13.01,
+      "tokens/trainable": 108647
+    },
+    {
+      "epoch": 0.11428571428571428,
+      "grad_norm": 0.253380686044693,
+      "learning_rate": 9.981568596205917e-05,
+      "loss": 0.30480891466140747,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 76.7,
+      "memory/max_allocated (GiB)": 76.7,
+      "ppl": 1.35637,
+      "step": 56,
+      "tokens/total": 13749568,
+      "tokens/train_per_sec_per_gpu": 11.0,
+      "tokens/trainable": 110620
+    },
+    {
+      "epoch": 0.11632653061224489,
+      "grad_norm": 0.41110867261886597,
+      "learning_rate": 9.980124488638774e-05,
+      "loss": 0.369600772857666,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 72.1,
+      "memory/max_allocated (GiB)": 72.1,
+      "ppl": 1.44716,
+      "step": 57,
+      "tokens/total": 13985344,
+      "tokens/train_per_sec_per_gpu": 15.88,
+      "tokens/trainable": 112743
+    },
+    {
+      "epoch": 0.11836734693877551,
+      "grad_norm": 0.3153546452522278,
+      "learning_rate": 9.978626033703732e-05,
+      "loss": 0.3381732702255249,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 73.43,
+      "memory/max_allocated (GiB)": 73.43,
+      "ppl": 1.40238,
+      "step": 58,
+      "tokens/total": 14196480,
+      "tokens/train_per_sec_per_gpu": 13.26,
+      "tokens/trainable": 114572
+    },
+    {
+      "epoch": 0.12040816326530612,
+      "grad_norm": 0.26562801003456116,
+      "learning_rate": 9.977073247753218e-05,
+      "loss": 0.3169305920600891,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 74.69,
+      "memory/max_allocated (GiB)": 74.69,
+      "ppl": 1.37291,
+      "step": 59,
+      "tokens/total": 14435008,
+      "tokens/train_per_sec_per_gpu": 12.93,
+      "tokens/trainable": 116707
+    },
+    {
+      "epoch": 0.12244897959183673,
+      "grad_norm": 0.3556833565235138,
+      "learning_rate": 9.97546614773255e-05,
+      "loss": 0.2778860330581665,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 74.39,
+      "memory/max_allocated (GiB)": 74.39,
+      "ppl": 1.32034,
+      "step": 60,
+      "tokens/total": 14648576,
+      "tokens/train_per_sec_per_gpu": 11.16,
+      "tokens/trainable": 118447
+    },
+    {
+      "epoch": 0.12448979591836734,
+      "grad_norm": 0.23345772922039032,
+      "learning_rate": 9.973804751179779e-05,
+      "loss": 0.27497315406799316,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 75.67,
+      "memory/max_allocated (GiB)": 75.67,
+      "ppl": 1.3165,
+      "step": 61,
+      "tokens/total": 14898368,
+      "tokens/train_per_sec_per_gpu": 12.07,
+      "tokens/trainable": 120410
+    },
+    {
+      "epoch": 0.12653061224489795,
+      "grad_norm": 0.2623974680900574,
+      "learning_rate": 9.972089076225477e-05,
+      "loss": 0.27318501472473145,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 75.34,
+      "memory/max_allocated (GiB)": 75.34,
+      "ppl": 1.31414,
+      "step": 62,
+      "tokens/total": 15121792,
+      "tokens/train_per_sec_per_gpu": 11.53,
+      "tokens/trainable": 122325
+    },
+    {
+      "epoch": 0.12857142857142856,
+      "grad_norm": 0.30083271861076355,
+      "learning_rate": 9.97031914159256e-05,
+      "loss": 0.3412373960018158,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 74.49,
+      "memory/max_allocated (GiB)": 74.49,
+      "ppl": 1.40669,
+      "step": 63,
+      "tokens/total": 15389312,
+      "tokens/train_per_sec_per_gpu": 12.31,
+      "tokens/trainable": 124224
+    },
+    {
+      "epoch": 0.1306122448979592,
+      "grad_norm": 0.2646200656890869,
+      "learning_rate": 9.968494966596058e-05,
+      "loss": 0.31369322538375854,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 73.95,
+      "memory/max_allocated (GiB)": 73.95,
+      "ppl": 1.36847,
+      "step": 64,
+      "tokens/total": 15630144,
+      "tokens/train_per_sec_per_gpu": 11.16,
+      "tokens/trainable": 126004
+    },
+    {
+      "epoch": 0.1326530612244898,
+      "grad_norm": 0.4355469048023224,
+      "learning_rate": 9.966616571142931e-05,
+      "loss": 0.335609495639801,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 76.32,
+      "memory/max_allocated (GiB)": 76.32,
+      "ppl": 1.39879,
+      "step": 65,
+      "tokens/total": 15912896,
+      "tokens/train_per_sec_per_gpu": 10.23,
+      "tokens/trainable": 128003
+    },
+    {
+      "epoch": 0.1346938775510204,
+      "grad_norm": 0.3418930172920227,
+      "learning_rate": 9.964683975731828e-05,
+      "loss": 0.31057706475257874,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 75.94,
+      "memory/max_allocated (GiB)": 75.94,
+      "ppl": 1.36421,
+      "step": 66,
+      "tokens/total": 16185728,
+      "tokens/train_per_sec_per_gpu": 12.43,
+      "tokens/trainable": 130139
+    },
+    {
+      "epoch": 0.13673469387755102,
+      "grad_norm": 0.27495577931404114,
+      "learning_rate": 9.962697201452881e-05,
+      "loss": 0.3229851722717285,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 76.32,
+      "memory/max_allocated (GiB)": 76.32,
+      "ppl": 1.38124,
+      "step": 67,
+      "tokens/total": 16449728,
+      "tokens/train_per_sec_per_gpu": 11.97,
+      "tokens/trainable": 132201
+    },
+    {
+      "epoch": 0.13877551020408163,
+      "grad_norm": 0.31963616609573364,
+      "learning_rate": 9.96065626998747e-05,
+      "loss": 0.3738800883293152,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 76.48,
+      "memory/max_allocated (GiB)": 76.48,
+      "ppl": 1.45336,
+      "step": 68,
+      "tokens/total": 16719488,
+      "tokens/train_per_sec_per_gpu": 9.86,
+      "tokens/trainable": 134160
+    },
+    {
+      "epoch": 0.14081632653061224,
+      "grad_norm": 0.29570698738098145,
+      "learning_rate": 9.958561203607975e-05,
+      "loss": 0.32448041439056396,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 73.57,
+      "memory/max_allocated (GiB)": 73.57,
+      "ppl": 1.38331,
+      "step": 69,
+      "tokens/total": 16953152,
+      "tokens/train_per_sec_per_gpu": 13.49,
+      "tokens/trainable": 136020
+    },
+    {
+      "epoch": 0.14285714285714285,
+      "grad_norm": 0.3492854833602905,
+      "learning_rate": 9.956412025177551e-05,
+      "loss": 0.35642874240875244,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 76.73,
+      "memory/max_allocated (GiB)": 76.73,
+      "ppl": 1.42822,
+      "step": 70,
+      "tokens/total": 17219072,
+      "tokens/train_per_sec_per_gpu": 11.64,
+      "tokens/trainable": 137981
+    },
+    {
+      "epoch": 0.14489795918367346,
+      "grad_norm": 0.24466252326965332,
+      "learning_rate": 9.954208758149867e-05,
+      "loss": 0.23536062240600586,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 73.92,
+      "memory/max_allocated (GiB)": 73.92,
+      "ppl": 1.26537,
+      "step": 71,
+      "tokens/total": 17490944,
+      "tokens/train_per_sec_per_gpu": 10.28,
+      "tokens/trainable": 139847
+    },
+    {
+      "epoch": 0.1469387755102041,
+      "grad_norm": 0.27268102765083313,
+      "learning_rate": 9.951951426568851e-05,
+      "loss": 0.2678673565387726,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 72.34,
+      "memory/max_allocated (GiB)": 72.34,
+      "ppl": 1.30717,
+      "step": 72,
+      "tokens/total": 17732032,
+      "tokens/train_per_sec_per_gpu": 13.63,
+      "tokens/trainable": 141879
+    },
+    {
+      "epoch": 0.1489795918367347,
+      "grad_norm": 0.33937132358551025,
+      "learning_rate": 9.949640055068432e-05,
+      "loss": 0.40895330905914307,
+      "memory/device_reserved (GiB)": 77.65,
+      "memory/max_active (GiB)": 72.29,
+      "memory/max_allocated (GiB)": 72.29,
+      "ppl": 1.50524,
+      "step": 73,
+      "tokens/total": 17969088,
+      "tokens/train_per_sec_per_gpu": 12.36,
+      "tokens/trainable": 143793
+    },
+    {
+      "epoch": 0.1510204081632653,
+      "grad_norm": 0.33928951621055603,
+      "learning_rate": 9.94727466887227e-05,
+      "loss": 0.3504130244255066,
+      "memory/device_reserved (GiB)": 77.75,
+      "memory/max_active (GiB)": 76.84,
+      "memory/max_allocated (GiB)": 76.84,
+      "ppl": 1.41965,
+      "step": 74,
+      "tokens/total": 18238784,
+      "tokens/train_per_sec_per_gpu": 11.17,
+      "tokens/trainable": 145834
+    },
+    {
+      "epoch": 0.15306122448979592,
+      "grad_norm": 0.23962773382663727,
+      "learning_rate": 9.944855293793475e-05,
+      "loss": 0.27072766423225403,
+      "memory/device_reserved (GiB)": 77.75,
+      "memory/max_active (GiB)": 76.51,
+      "memory/max_allocated (GiB)": 76.51,
+      "ppl": 1.31092,
+      "step": 75,
+      "tokens/total": 18495168,
+      "tokens/train_per_sec_per_gpu": 11.78,
+      "tokens/trainable": 147888
+    },
+    {
+      "epoch": 0.15510204081632653,
+      "grad_norm": 0.25025618076324463,
+      "learning_rate": 9.942381956234335e-05,
+      "loss": 0.28914934396743774,
+      "memory/device_reserved (GiB)": 77.75,
+      "memory/max_active (GiB)": 76.4,
+      "memory/max_allocated (GiB)": 76.4,
+      "ppl": 1.33529,
+      "step": 76,
+      "tokens/total": 18734016,
+      "tokens/train_per_sec_per_gpu": 9.17,
+      "tokens/trainable": 149764
+    },
+    {
+      "epoch": 0.15714285714285714,
+      "grad_norm": 0.26084867119789124,
+      "learning_rate": 9.939854683186016e-05,
+      "loss": 0.3206363916397095,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 73.62,
+      "memory/max_allocated (GiB)": 73.62,
+      "ppl": 1.378,
+      "step": 77,
+      "tokens/total": 18972672,
+      "tokens/train_per_sec_per_gpu": 12.82,
+      "tokens/trainable": 151857
+    },
+    {
+      "epoch": 0.15918367346938775,
+      "grad_norm": 0.29429370164871216,
+      "learning_rate": 9.937273502228281e-05,
+      "loss": 0.3649798631668091,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 75.8,
+      "memory/max_allocated (GiB)": 75.8,
+      "ppl": 1.44049,
+      "step": 78,
+      "tokens/total": 19195392,
+      "tokens/train_per_sec_per_gpu": 13.09,
+      "tokens/trainable": 153856
+    },
+    {
+      "epoch": 0.16122448979591836,
+      "grad_norm": 0.32660233974456787,
+      "learning_rate": 9.934638441529179e-05,
+      "loss": 0.35134780406951904,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 74.06,
+      "memory/max_allocated (GiB)": 74.06,
+      "ppl": 1.42098,
+      "step": 79,
+      "tokens/total": 19472064,
+      "tokens/train_per_sec_per_gpu": 9.99,
+      "tokens/trainable": 155739
+    },
+    {
+      "epoch": 0.16326530612244897,
+      "grad_norm": 0.3226783573627472,
+      "learning_rate": 9.93194952984474e-05,
+      "loss": 0.386910617351532,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 74.06,
+      "memory/max_allocated (GiB)": 74.06,
+      "ppl": 1.47242,
+      "step": 80,
+      "tokens/total": 19724544,
+      "tokens/train_per_sec_per_gpu": 13.41,
+      "tokens/trainable": 158060
+    },
+    {
+      "epoch": 0.1653061224489796,
+      "grad_norm": 0.3477247655391693,
+      "learning_rate": 9.929206796518662e-05,
+      "loss": 0.3311561346054077,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 71.64,
+      "memory/max_allocated (GiB)": 71.64,
+      "ppl": 1.39258,
+      "step": 81,
+      "tokens/total": 19941440,
+      "tokens/train_per_sec_per_gpu": 14.15,
+      "tokens/trainable": 159968
+    },
+    {
+      "epoch": 0.1673469387755102,
+      "grad_norm": 0.2438858300447464,
+      "learning_rate": 9.926410271481994e-05,
+      "loss": 0.297912061214447,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 74.52,
+      "memory/max_allocated (GiB)": 74.52,
+      "ppl": 1.34704,
+      "step": 82,
+      "tokens/total": 20194560,
+      "tokens/train_per_sec_per_gpu": 10.91,
+      "tokens/trainable": 161784
+    },
+    {
+      "epoch": 0.16938775510204082,
+      "grad_norm": 0.38258686661720276,
+      "learning_rate": 9.9235599852528e-05,
+      "loss": 0.2937951982021332,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 75.09,
+      "memory/max_allocated (GiB)": 75.09,
+      "ppl": 1.34151,
+      "step": 83,
+      "tokens/total": 20438272,
+      "tokens/train_per_sec_per_gpu": 12.73,
+      "tokens/trainable": 163745
+    },
+    {
+      "epoch": 0.17142857142857143,
+      "grad_norm": 0.2722567021846771,
+      "learning_rate": 9.920655968935838e-05,
+      "loss": 0.31319552659988403,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 73.62,
+      "memory/max_allocated (GiB)": 73.62,
+      "ppl": 1.36779,
+      "step": 84,
+      "tokens/total": 20647488,
+      "tokens/train_per_sec_per_gpu": 12.15,
+      "tokens/trainable": 165491
+    },
+    {
+      "epoch": 0.17346938775510204,
+      "grad_norm": 0.22799928486347198,
+      "learning_rate": 9.91769825422221e-05,
+      "loss": 0.2948286533355713,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 73.54,
+      "memory/max_allocated (GiB)": 73.54,
+      "ppl": 1.3429,
+      "step": 85,
+      "tokens/total": 20908352,
+      "tokens/train_per_sec_per_gpu": 11.91,
+      "tokens/trainable": 167486
+    },
+    {
+      "epoch": 0.17551020408163265,
+      "grad_norm": 0.29396697878837585,
+      "learning_rate": 9.914686873389025e-05,
+      "loss": 0.3924490213394165,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 73.95,
+      "memory/max_allocated (GiB)": 73.95,
+      "ppl": 1.4806,
+      "step": 86,
+      "tokens/total": 21166656,
+      "tokens/train_per_sec_per_gpu": 12.61,
+      "tokens/trainable": 169501
+    },
+    {
+      "epoch": 0.17755102040816326,
+      "grad_norm": 0.4080873429775238,
+      "learning_rate": 9.91162185929904e-05,
+      "loss": 0.36466801166534424,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 76.35,
+      "memory/max_allocated (GiB)": 76.35,
+      "ppl": 1.44004,
+      "step": 87,
+      "tokens/total": 21438912,
+      "tokens/train_per_sec_per_gpu": 11.23,
+      "tokens/trainable": 171575
+    },
+    {
+      "epoch": 0.17959183673469387,
+      "grad_norm": 0.37823009490966797,
+      "learning_rate": 9.908503245400302e-05,
+      "loss": 0.36650943756103516,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 75.56,
+      "memory/max_allocated (GiB)": 75.56,
+      "ppl": 1.44269,
+      "step": 88,
+      "tokens/total": 21722176,
+      "tokens/train_per_sec_per_gpu": 12.08,
+      "tokens/trainable": 173659
+    },
+    {
+      "epoch": 0.1816326530612245,
+      "grad_norm": 0.30847790837287903,
+      "learning_rate": 9.90533106572579e-05,
+      "loss": 0.37945014238357544,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 74.11,
+      "memory/max_allocated (GiB)": 74.11,
+      "ppl": 1.46148,
+      "step": 89,
+      "tokens/total": 21958848,
+      "tokens/train_per_sec_per_gpu": 13.82,
+      "tokens/trainable": 175607
+    },
+    {
+      "epoch": 0.1836734693877551,
+      "grad_norm": 0.25030240416526794,
+      "learning_rate": 9.90210535489303e-05,
+      "loss": 0.25757214426994324,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 72.89,
+      "memory/max_allocated (GiB)": 72.89,
+      "ppl": 1.29379,
+      "step": 90,
+      "tokens/total": 22210112,
+      "tokens/train_per_sec_per_gpu": 13.01,
+      "tokens/trainable": 177797
+    },
+    {
+      "epoch": 0.18571428571428572,
+      "grad_norm": 0.2805105447769165,
+      "learning_rate": 9.898826148103737e-05,
+      "loss": 0.36492833495140076,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 75.34,
+      "memory/max_allocated (GiB)": 75.34,
+      "ppl": 1.44041,
+      "step": 91,
+      "tokens/total": 22450496,
+      "tokens/train_per_sec_per_gpu": 11.15,
+      "tokens/trainable": 179665
+    },
+    {
+      "epoch": 0.18775510204081633,
+      "grad_norm": 0.2761736512184143,
+      "learning_rate": 9.895493481143411e-05,
+      "loss": 0.3230007290840149,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 74.03,
+      "memory/max_allocated (GiB)": 74.03,
+      "ppl": 1.38127,
+      "step": 92,
+      "tokens/total": 22683776,
+      "tokens/train_per_sec_per_gpu": 13.16,
+      "tokens/trainable": 181685
+    },
+    {
+      "epoch": 0.18979591836734694,
+      "grad_norm": 0.2371329665184021,
+      "learning_rate": 9.892107390380958e-05,
+      "loss": 0.2528485655784607,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 72.37,
+      "memory/max_allocated (GiB)": 72.37,
+      "ppl": 1.28769,
+      "step": 93,
+      "tokens/total": 22917888,
+      "tokens/train_per_sec_per_gpu": 12.17,
+      "tokens/trainable": 183472
+    },
+    {
+      "epoch": 0.19183673469387755,
+      "grad_norm": 0.28814423084259033,
+      "learning_rate": 9.888667912768288e-05,
+      "loss": 0.3587278723716736,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 74.39,
+      "memory/max_allocated (GiB)": 74.39,
+      "ppl": 1.43151,
+      "step": 94,
+      "tokens/total": 23187520,
+      "tokens/train_per_sec_per_gpu": 10.7,
+      "tokens/trainable": 185479
+    },
+    {
+      "epoch": 0.19387755102040816,
+      "grad_norm": 0.24956394731998444,
+      "learning_rate": 9.885175085839919e-05,
+      "loss": 0.3131387233734131,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 74.47,
+      "memory/max_allocated (GiB)": 74.47,
+      "ppl": 1.36771,
+      "step": 95,
+      "tokens/total": 23443008,
+      "tokens/train_per_sec_per_gpu": 13.7,
+      "tokens/trainable": 187567
+    },
+    {
+      "epoch": 0.19591836734693877,
+      "grad_norm": 0.3290642201900482,
+      "learning_rate": 9.881628947712556e-05,
+      "loss": 0.2742847204208374,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 73.79,
+      "memory/max_allocated (GiB)": 73.79,
+      "ppl": 1.31559,
+      "step": 96,
+      "tokens/total": 23698432,
+      "tokens/train_per_sec_per_gpu": 9.94,
+      "tokens/trainable": 189220
+    },
+    {
+      "epoch": 0.19795918367346937,
+      "grad_norm": 0.27119094133377075,
+      "learning_rate": 9.878029537084684e-05,
+      "loss": 0.2874329686164856,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 74.82,
+      "memory/max_allocated (GiB)": 74.82,
+      "ppl": 1.333,
+      "step": 97,
+      "tokens/total": 23947968,
+      "tokens/train_per_sec_per_gpu": 10.06,
+      "tokens/trainable": 190943
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.28361037373542786,
+      "learning_rate": 9.874376893236145e-05,
+      "loss": 0.33660027384757996,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 73.79,
+      "memory/max_allocated (GiB)": 73.79,
+      "ppl": 1.40018,
+      "step": 98,
+      "tokens/total": 24192704,
+      "tokens/train_per_sec_per_gpu": 12.95,
+      "tokens/trainable": 192967
+    },
+    {
+      "epoch": 0.20204081632653062,
+      "grad_norm": 0.3728221654891968,
+      "learning_rate": 9.870671056027705e-05,
+      "loss": 0.30863887071609497,
+      "memory/device_reserved (GiB)": 77.31,
+      "memory/max_active (GiB)": 73.3,
+      "memory/max_allocated (GiB)": 73.3,
+      "ppl": 1.36157,
+      "step": 99,
+      "tokens/total": 24420672,
+      "tokens/train_per_sec_per_gpu": 13.93,
+      "tokens/trainable": 194997
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 0.24369771778583527,
+      "learning_rate": 9.866912065900621e-05,
+      "loss": 0.28639698028564453,
+      "memory/device_reserved (GiB)": 77.53,
+      "memory/max_active (GiB)": 76.51,
+      "memory/max_allocated (GiB)": 76.51,
+      "ppl": 1.33162,
+      "step": 100,
+      "tokens/total": 24663168,
+      "tokens/train_per_sec_per_gpu": 13.19,
+      "tokens/trainable": 197125
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 980,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 25,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.5671460522216653e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd3f808e28e895b0ab95910fe9e05e80cd659a3b5ece764c427bd3c13ec25562
+size 6673