Training in progress, step 500

BluebrainAI___wikitext-103-raw-v1-seq512-tokenized-grouped/default/0.0.0/f1720895be296a4e1af5c3f0902492373ed73d6e/dataset_info.json

@@ -0,0 +1 @@
+{"description": "", "citation": "", "homepage": "", "license": "", "features": {"input_ids": {"feature": {"dtype": "int32", "_type": "Value"}, "_type": "Sequence"}, "attention_mask": {"feature": {"dtype": "int8", "_type": "Value"}, "_type": "Sequence"}}, "builder_name": "parquet", "dataset_name": "wikitext-103-raw-v1-seq512-tokenized-grouped", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"validation": {"name": "validation", "num_bytes": 6006224, "num_examples": 2282, "dataset_name": "wikitext-103-raw-v1-seq512-tokenized-grouped"}, "train": {"name": "train", "num_bytes": 600111792, "num_examples": 228006, "shard_lengths": [190000, 38006], "dataset_name": "wikitext-103-raw-v1-seq512-tokenized-grouped"}}, "download_checksums": {"hf://datasets/BluebrainAI/wikitext-103-raw-v1-seq512-tokenized-grouped@f1720895be296a4e1af5c3f0902492373ed73d6e/data/validation-00000-of-00001.parquet": {"num_bytes": 2404607, "checksum": null}, "hf://datasets/BluebrainAI/wikitext-103-raw-v1-seq512-tokenized-grouped@f1720895be296a4e1af5c3f0902492373ed73d6e/data/train-00000-of-00001.parquet": {"num_bytes": 240278980, "checksum": null}}, "download_size": 242683587, "dataset_size": 606118016, "size_in_bytes": 848801603}

BluebrainAI___wikitext-103-raw-v1-seq512-tokenized-grouped/default/0.0.0/f1720895be296a4e1af5c3f0902492373ed73d6e/wikitext-103-raw-v1-seq512-tokenized-grouped-train-00000-of-00002.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac7dd270748aa68ba0688a8ca5740ca8dc774fb94d4ec35cbce6a29c9fcfae3
+size 487978312

BluebrainAI___wikitext-103-raw-v1-seq512-tokenized-grouped/default/0.0.0/f1720895be296a4e1af5c3f0902492373ed73d6e/wikitext-103-raw-v1-seq512-tokenized-grouped-train-00001-of-00002.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59a00692201253b3f1d8f77665e7a818678e78335786f2721a9ec91a81566319
+size 97611816

BluebrainAI___wikitext-103-raw-v1-seq512-tokenized-grouped/default/0.0.0/f1720895be296a4e1af5c3f0902492373ed73d6e/wikitext-103-raw-v1-seq512-tokenized-grouped-validation.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90ed4e155cf7fd90b36ae18c475af8a1aa8f2e02b8a77912f2876f74c3637754
+size 5861640

all_results.json

@@ -1,17 +1,17 @@
 {
     "epoch": 5.0,
-    "eval_accuracy": 0.02236511042773274,
-    "eval_bleu": 0.002799146980613368,
-    "eval_loss": 9.125165939331055,
-    "eval_perplexity": 9183.52079846965,
-    "eval_runtime": 0.3917,
-    "eval_samples": 14,
-    "eval_samples_per_second": 35.745,
-    "eval_steps_per_second": 2.553,
-    "perplexity": 9183.52079846965,
-    "total_flos": 129891126435840.0,
-    "train_loss": 19.52422180175781,
-    "train_runtime": 23.5998,
-    "train_samples_per_second": 2.966,
-    "train_steps_per_second": 0.212
+    "eval_accuracy": 0.00733640796431187,
+    "eval_bleu": 1.0,
+    "eval_loss": 1.5712791681289673,
+    "eval_perplexity": 4.8128006391188904,
+    "eval_runtime": 30.7765,
+    "eval_samples": 2282,
+    "eval_samples_per_second": 74.148,
+    "eval_steps_per_second": 1.17,
+    "perplexity": 4.8128006391188904,
+    "total_flos": 2.1154254410092954e+18,
+    "train_loss": 3.7582801998715505,
+    "train_runtime": 52733.3783,
+    "train_samples_per_second": 21.619,
+    "train_steps_per_second": 0.338
 }

eval_metrics.json

@@ -0,0 +1,10 @@
+{
+    "eval_loss": 23.060916900634766,
+    "eval_model_preparation_time": 0.0165,
+    "eval_accuracy": 0.00733640796431187,
+    "eval_perplexity": 10356880316.044792,
+    "eval_bleu": 1.0,
+    "eval_runtime": 11.0041,
+    "eval_samples_per_second": 207.378,
+    "eval_steps_per_second": 3.272
+}

eval_results.json

@@ -1,12 +1,12 @@
 {
     "epoch": 5.0,
-    "eval_accuracy": 0.02236511042773274,
-    "eval_bleu": 0.002799146980613368,
-    "eval_loss": 9.125165939331055,
-    "eval_perplexity": 9183.52079846965,
-    "eval_runtime": 0.3917,
-    "eval_samples": 14,
-    "eval_samples_per_second": 35.745,
-    "eval_steps_per_second": 2.553,
-    "perplexity": 9183.52079846965
+    "eval_accuracy": 0.00733640796431187,
+    "eval_bleu": 1.0,
+    "eval_loss": 1.5712791681289673,
+    "eval_perplexity": 4.8128006391188904,
+    "eval_runtime": 30.7765,
+    "eval_samples": 2282,
+    "eval_samples_per_second": 74.148,
+    "eval_steps_per_second": 1.17,
+    "perplexity": 4.8128006391188904
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:897353a015a9528d9cd7e7ac2486bf757a8623e3d4fd6c5fabd2e3f596fdfd16
+oid sha256:71f02f38d4d8f10224f5b9b54a9637315197efa4c2f98fb81c52d270ad331392
 size 1417229824

modeling_duo_predict_gpt2.py

@@ -129,10 +129,10 @@ def sdpa_attention_forward(
         query,
         key,
         value,
-        attn_mask=create_attention_mask_matrix(query.shape[-2]).to(query.device) if module.training else None,
+        attn_mask=create_attention_mask_matrix(query.shape[-2]).to(query.device),
         dropout_p=dropout,
         scale=scaling,
-        is_causal=False if module.training else True,
+        is_causal=is_causal,
     )
     attn_output = attn_output.transpose(1, 2).contiguous()
 
@@ -582,12 +582,9 @@ class DuoPredictGPT2Model(DuoPredictGPT2PretrainedModel):
             inputs_embeds = self.wte(input_ids)
         position_embeds = self.wpe(position_ids)
         ###TODO: correctly initialized
-        if inputs_embeds.shape[1] != position_embeds.shape[1]:
-            hidden_states = torch.empty((batch_size, input_shape[-1], self.embed_dim), device=device)
-            hidden_states[:, ::2] = inputs_embeds[:, ::2] + position_embeds.to(inputs_embeds.device)
-            hidden_states[:, 1::2] = inputs_embeds[:, 1::2] + position_embeds[:, :self.config.max_position_embeddings-1].to(inputs_embeds.device)
-        else:
-            hidden_states = inputs_embeds + position_embeds
+        hidden_states = torch.empty((batch_size, input_shape[-1], self.embed_dim), device=device)
+        hidden_states[:, ::2] = inputs_embeds[:, ::2] + position_embeds.to(inputs_embeds.device)
+        hidden_states[:, 1::2] = inputs_embeds[:, 1::2] + position_embeds[:, :self.config.max_position_embeddings-1].to(inputs_embeds.device)
 
         # Attention mask.
         _use_sdpa = self._attn_implementation == "sdpa" and output_attentions is False and head_mask is None
@@ -900,6 +897,5 @@ if __name__  == "__main__":
     model = DuoPredictGPT2LMHeadModel(cg)
     from src.utils.model_utlis import print_trainable_parameters
     print_trainable_parameters(model)
-    model.eval()
     model(torch.randint(0, 10000, (1, 100)))
     print()

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 5.0,
-    "total_flos": 129891126435840.0,
-    "train_loss": 19.52422180175781,
-    "train_runtime": 23.5998,
-    "train_samples_per_second": 2.966,
-    "train_steps_per_second": 0.212
+    "total_flos": 2.1154254410092954e+18,
+    "train_loss": 3.7582801998715505,
+    "train_runtime": 52733.3783,
+    "train_samples_per_second": 21.619,
+    "train_steps_per_second": 0.338
 }

trainer_state.json

@@ -1,25 +1,1656 @@
 {
-  "best_metric": null,
-  "best_model_checkpoint": null,
+  "best_metric": 1.5712791681289673,
+  "best_model_checkpoint": "./output/models/duo-predict-gpt2-medium-wikitext/checkpoint-17500",
   "epoch": 5.0,
   "eval_steps": 500,
-  "global_step": 5,
+  "global_step": 17815,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
+    {
+      "epoch": 0.028066236317709794,
+      "grad_norm": 18.272640228271484,
+      "learning_rate": 5.611672278338946e-06,
+      "loss": 16.6534,
+      "step": 100
+    },
+    {
+      "epoch": 0.05613247263541959,
+      "grad_norm": 11.887374877929688,
+      "learning_rate": 1.1223344556677892e-05,
+      "loss": 11.6666,
+      "step": 200
+    },
+    {
+      "epoch": 0.08419870895312939,
+      "grad_norm": 10.138073921203613,
+      "learning_rate": 1.6835016835016837e-05,
+      "loss": 9.3278,
+      "step": 300
+    },
+    {
+      "epoch": 0.11226494527083918,
+      "grad_norm": 6.484872817993164,
+      "learning_rate": 2.2446689113355783e-05,
+      "loss": 8.1162,
+      "step": 400
+    },
+    {
+      "epoch": 0.140331181588549,
+      "grad_norm": 3.6428794860839844,
+      "learning_rate": 2.8058361391694725e-05,
+      "loss": 7.6316,
+      "step": 500
+    },
+    {
+      "epoch": 0.140331181588549,
+      "eval_accuracy": 0.007349271333039477,
+      "eval_bleu": 1.0,
+      "eval_loss": 3.704052448272705,
+      "eval_perplexity": 40.61154753747588,
+      "eval_runtime": 30.7567,
+      "eval_samples_per_second": 74.195,
+      "eval_steps_per_second": 1.17,
+      "step": 500
+    },
+    {
+      "epoch": 0.16839741790625878,
+      "grad_norm": 3.5977585315704346,
+      "learning_rate": 3.3670033670033675e-05,
+      "loss": 7.4262,
+      "step": 600
+    },
+    {
+      "epoch": 0.19646365422396855,
+      "grad_norm": 3.0089902877807617,
+      "learning_rate": 3.9281705948372613e-05,
+      "loss": 7.2448,
+      "step": 700
+    },
+    {
+      "epoch": 0.22452989054167835,
+      "grad_norm": 3.010714530944824,
+      "learning_rate": 4.4893378226711566e-05,
+      "loss": 7.0773,
+      "step": 800
+    },
+    {
+      "epoch": 0.25259612685938815,
+      "grad_norm": 3.661517858505249,
+      "learning_rate": 5.050505050505051e-05,
+      "loss": 6.8409,
+      "step": 900
+    },
+    {
+      "epoch": 0.280662363177098,
+      "grad_norm": 2.2237136363983154,
+      "learning_rate": 5.611672278338945e-05,
+      "loss": 6.4879,
+      "step": 1000
+    },
+    {
+      "epoch": 0.280662363177098,
+      "eval_accuracy": 0.007337265522227044,
+      "eval_bleu": 0.9995095586630022,
+      "eval_loss": 3.119645595550537,
+      "eval_perplexity": 22.638355087518736,
+      "eval_runtime": 30.8837,
+      "eval_samples_per_second": 73.89,
+      "eval_steps_per_second": 1.166,
+      "step": 1000
+    },
+    {
+      "epoch": 0.30872859949480774,
+      "grad_norm": 2.7749340534210205,
+      "learning_rate": 6.17283950617284e-05,
+      "loss": 6.1614,
+      "step": 1100
+    },
+    {
+      "epoch": 0.33679483581251757,
+      "grad_norm": 1.8694963455200195,
+      "learning_rate": 6.734006734006735e-05,
+      "loss": 5.8679,
+      "step": 1200
+    },
+    {
+      "epoch": 0.36486107213022734,
+      "grad_norm": 1.7338366508483887,
+      "learning_rate": 7.295173961840629e-05,
+      "loss": 5.6537,
+      "step": 1300
+    },
+    {
+      "epoch": 0.3929273084479371,
+      "grad_norm": 1.5872080326080322,
+      "learning_rate": 7.856341189674523e-05,
+      "loss": 5.4812,
+      "step": 1400
+    },
+    {
+      "epoch": 0.42099354476564693,
+      "grad_norm": 1.8420644998550415,
+      "learning_rate": 8.417508417508418e-05,
+      "loss": 5.3189,
+      "step": 1500
+    },
+    {
+      "epoch": 0.42099354476564693,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 2.5976459980010986,
+      "eval_perplexity": 13.43208164297786,
+      "eval_runtime": 30.857,
+      "eval_samples_per_second": 73.954,
+      "eval_steps_per_second": 1.167,
+      "step": 1500
+    },
+    {
+      "epoch": 0.4490597810833567,
+      "grad_norm": 1.4730021953582764,
+      "learning_rate": 8.978675645342313e-05,
+      "loss": 5.1876,
+      "step": 1600
+    },
+    {
+      "epoch": 0.4771260174010665,
+      "grad_norm": 1.5234594345092773,
+      "learning_rate": 9.539842873176207e-05,
+      "loss": 5.0777,
+      "step": 1700
+    },
+    {
+      "epoch": 0.5051922537187763,
+      "grad_norm": 1.2363477945327759,
+      "learning_rate": 9.988773155367056e-05,
+      "loss": 4.9567,
+      "step": 1800
+    },
+    {
+      "epoch": 0.5332584900364861,
+      "grad_norm": 1.2880409955978394,
+      "learning_rate": 9.926401796295142e-05,
+      "loss": 4.8629,
+      "step": 1900
+    },
+    {
+      "epoch": 0.561324726354196,
+      "grad_norm": 1.1858636140823364,
+      "learning_rate": 9.864030437223227e-05,
+      "loss": 4.7557,
+      "step": 2000
+    },
+    {
+      "epoch": 0.561324726354196,
+      "eval_accuracy": 0.007337265522227044,
+      "eval_bleu": 1.0,
+      "eval_loss": 2.3368639945983887,
+      "eval_perplexity": 10.348731942766573,
+      "eval_runtime": 30.8164,
+      "eval_samples_per_second": 74.051,
+      "eval_steps_per_second": 1.168,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5893909626719057,
+      "grad_norm": 1.1543203592300415,
+      "learning_rate": 9.801659078151312e-05,
+      "loss": 4.6862,
+      "step": 2100
+    },
+    {
+      "epoch": 0.6174571989896155,
+      "grad_norm": 1.0767921209335327,
+      "learning_rate": 9.739287719079399e-05,
+      "loss": 4.6014,
+      "step": 2200
+    },
+    {
+      "epoch": 0.6455234353073253,
+      "grad_norm": 1.0958943367004395,
+      "learning_rate": 9.676916360007485e-05,
+      "loss": 4.5195,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6735896716250351,
+      "grad_norm": 1.1153371334075928,
+      "learning_rate": 9.61454500093557e-05,
+      "loss": 4.4267,
+      "step": 2400
+    },
+    {
+      "epoch": 0.7016559079427449,
+      "grad_norm": 1.1394137144088745,
+      "learning_rate": 9.552173641863657e-05,
+      "loss": 4.351,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7016559079427449,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 2.1233675479888916,
+      "eval_perplexity": 8.359240285181098,
+      "eval_runtime": 30.8157,
+      "eval_samples_per_second": 74.053,
+      "eval_steps_per_second": 1.168,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7297221442604547,
+      "grad_norm": 1.0487689971923828,
+      "learning_rate": 9.489802282791742e-05,
+      "loss": 4.2704,
+      "step": 2600
+    },
+    {
+      "epoch": 0.7577883805781644,
+      "grad_norm": 1.0785913467407227,
+      "learning_rate": 9.427430923719829e-05,
+      "loss": 4.2184,
+      "step": 2700
+    },
+    {
+      "epoch": 0.7858546168958742,
+      "grad_norm": 1.016829490661621,
+      "learning_rate": 9.365059564647915e-05,
+      "loss": 4.1661,
+      "step": 2800
+    },
+    {
+      "epoch": 0.8139208532135841,
+      "grad_norm": 1.0146821737289429,
+      "learning_rate": 9.302688205576e-05,
+      "loss": 4.1221,
+      "step": 2900
+    },
+    {
+      "epoch": 0.8419870895312939,
+      "grad_norm": 1.0396431684494019,
+      "learning_rate": 9.240316846504086e-05,
+      "loss": 4.0701,
+      "step": 3000
+    },
+    {
+      "epoch": 0.8419870895312939,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 2.002086877822876,
+      "eval_perplexity": 7.404492257321689,
+      "eval_runtime": 30.948,
+      "eval_samples_per_second": 73.736,
+      "eval_steps_per_second": 1.163,
+      "step": 3000
+    },
+    {
+      "epoch": 0.8700533258490036,
+      "grad_norm": 1.023743987083435,
+      "learning_rate": 9.177945487432171e-05,
+      "loss": 4.024,
+      "step": 3100
+    },
+    {
+      "epoch": 0.8981195621667134,
+      "grad_norm": 1.0103009939193726,
+      "learning_rate": 9.115574128360257e-05,
+      "loss": 4.0087,
+      "step": 3200
+    },
+    {
+      "epoch": 0.9261857984844233,
+      "grad_norm": 0.9384715557098389,
+      "learning_rate": 9.053202769288343e-05,
+      "loss": 3.9665,
+      "step": 3300
+    },
+    {
+      "epoch": 0.954252034802133,
+      "grad_norm": 0.9419653415679932,
+      "learning_rate": 8.990831410216429e-05,
+      "loss": 3.9352,
+      "step": 3400
+    },
+    {
+      "epoch": 0.9823182711198428,
+      "grad_norm": 0.9483797550201416,
+      "learning_rate": 8.928460051144514e-05,
+      "loss": 3.9112,
+      "step": 3500
+    },
+    {
+      "epoch": 0.9823182711198428,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.9279801845550537,
+      "eval_perplexity": 6.8756087463407365,
+      "eval_runtime": 30.8769,
+      "eval_samples_per_second": 73.906,
+      "eval_steps_per_second": 1.166,
+      "step": 3500
+    },
+    {
+      "epoch": 1.0103845074375526,
+      "grad_norm": 0.9196979403495789,
+      "learning_rate": 8.866088692072601e-05,
+      "loss": 3.8761,
+      "step": 3600
+    },
+    {
+      "epoch": 1.0384507437552624,
+      "grad_norm": 0.8850469589233398,
+      "learning_rate": 8.803717333000687e-05,
+      "loss": 3.8219,
+      "step": 3700
+    },
+    {
+      "epoch": 1.0665169800729721,
+      "grad_norm": 0.8743171095848083,
+      "learning_rate": 8.741345973928773e-05,
+      "loss": 3.8114,
+      "step": 3800
+    },
+    {
+      "epoch": 1.094583216390682,
+      "grad_norm": 0.8881787657737732,
+      "learning_rate": 8.678974614856859e-05,
+      "loss": 3.7866,
+      "step": 3900
+    },
+    {
+      "epoch": 1.122649452708392,
+      "grad_norm": 0.9794315695762634,
+      "learning_rate": 8.616603255784944e-05,
+      "loss": 3.7775,
+      "step": 4000
+    },
+    {
+      "epoch": 1.122649452708392,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.87688148021698,
+      "eval_perplexity": 6.533099461490326,
+      "eval_runtime": 30.8839,
+      "eval_samples_per_second": 73.89,
+      "eval_steps_per_second": 1.166,
+      "step": 4000
+    },
+    {
+      "epoch": 1.1507156890261017,
+      "grad_norm": 0.8849213719367981,
+      "learning_rate": 8.55423189671303e-05,
+      "loss": 3.761,
+      "step": 4100
+    },
+    {
+      "epoch": 1.1787819253438114,
+      "grad_norm": 0.9264663457870483,
+      "learning_rate": 8.491860537641115e-05,
+      "loss": 3.7411,
+      "step": 4200
+    },
+    {
+      "epoch": 1.2068481616615212,
+      "grad_norm": 0.8461829423904419,
+      "learning_rate": 8.429489178569201e-05,
+      "loss": 3.7261,
+      "step": 4300
+    },
+    {
+      "epoch": 1.234914397979231,
+      "grad_norm": 0.9400424957275391,
+      "learning_rate": 8.367117819497288e-05,
+      "loss": 3.7196,
+      "step": 4400
+    },
+    {
+      "epoch": 1.2629806342969407,
+      "grad_norm": 0.8834152817726135,
+      "learning_rate": 8.304746460425373e-05,
+      "loss": 3.703,
+      "step": 4500
+    },
+    {
+      "epoch": 1.2629806342969407,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.8368678092956543,
+      "eval_perplexity": 6.276847156661428,
+      "eval_runtime": 30.8747,
+      "eval_samples_per_second": 73.912,
+      "eval_steps_per_second": 1.166,
+      "step": 4500
+    },
+    {
+      "epoch": 1.2910468706146505,
+      "grad_norm": 0.8625416159629822,
+      "learning_rate": 8.242375101353458e-05,
+      "loss": 3.6845,
+      "step": 4600
+    },
+    {
+      "epoch": 1.3191131069323605,
+      "grad_norm": 0.8428071141242981,
+      "learning_rate": 8.180003742281544e-05,
+      "loss": 3.6765,
+      "step": 4700
+    },
+    {
+      "epoch": 1.3471793432500703,
+      "grad_norm": 0.8716861605644226,
+      "learning_rate": 8.117632383209631e-05,
+      "loss": 3.6565,
+      "step": 4800
+    },
+    {
+      "epoch": 1.37524557956778,
+      "grad_norm": 0.878508985042572,
+      "learning_rate": 8.055261024137716e-05,
+      "loss": 3.6412,
+      "step": 4900
+    },
+    {
+      "epoch": 1.4033118158854898,
+      "grad_norm": 0.8190825581550598,
+      "learning_rate": 7.992889665065803e-05,
+      "loss": 3.6443,
+      "step": 5000
+    },
+    {
+      "epoch": 1.4033118158854898,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.8060888051986694,
+      "eval_perplexity": 6.086594958242813,
+      "eval_runtime": 30.8156,
+      "eval_samples_per_second": 74.053,
+      "eval_steps_per_second": 1.168,
+      "step": 5000
+    },
+    {
+      "epoch": 1.4313780522031996,
+      "grad_norm": 0.8404738903045654,
+      "learning_rate": 7.930518305993888e-05,
+      "loss": 3.6201,
+      "step": 5100
+    },
+    {
+      "epoch": 1.4594442885209093,
+      "grad_norm": 0.8295940160751343,
+      "learning_rate": 7.868146946921974e-05,
+      "loss": 3.6238,
+      "step": 5200
+    },
+    {
+      "epoch": 1.4875105248386191,
+      "grad_norm": 0.8289036750793457,
+      "learning_rate": 7.80577558785006e-05,
+      "loss": 3.605,
+      "step": 5300
+    },
+    {
+      "epoch": 1.5155767611563289,
+      "grad_norm": 0.8368122577667236,
+      "learning_rate": 7.743404228778145e-05,
+      "loss": 3.5954,
+      "step": 5400
+    },
+    {
+      "epoch": 1.5436429974740387,
+      "grad_norm": 0.8075991272926331,
+      "learning_rate": 7.681032869706232e-05,
+      "loss": 3.5659,
+      "step": 5500
+    },
+    {
+      "epoch": 1.5436429974740387,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.7788798809051514,
+      "eval_perplexity": 5.9232179917953625,
+      "eval_runtime": 30.8647,
+      "eval_samples_per_second": 73.936,
+      "eval_steps_per_second": 1.166,
+      "step": 5500
+    },
+    {
+      "epoch": 1.5717092337917484,
+      "grad_norm": 0.8155397176742554,
+      "learning_rate": 7.618661510634317e-05,
+      "loss": 3.5588,
+      "step": 5600
+    },
+    {
+      "epoch": 1.5997754701094582,
+      "grad_norm": 0.8227369785308838,
+      "learning_rate": 7.556290151562403e-05,
+      "loss": 3.5663,
+      "step": 5700
+    },
+    {
+      "epoch": 1.627841706427168,
+      "grad_norm": 0.8452321290969849,
+      "learning_rate": 7.493918792490488e-05,
+      "loss": 3.5497,
+      "step": 5800
+    },
+    {
+      "epoch": 1.655907942744878,
+      "grad_norm": 0.8576854467391968,
+      "learning_rate": 7.431547433418574e-05,
+      "loss": 3.541,
+      "step": 5900
+    },
+    {
+      "epoch": 1.6839741790625877,
+      "grad_norm": 0.8500741720199585,
+      "learning_rate": 7.36917607434666e-05,
+      "loss": 3.5303,
+      "step": 6000
+    },
+    {
+      "epoch": 1.6839741790625877,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.7552289962768555,
+      "eval_perplexity": 5.784772281608607,
+      "eval_runtime": 30.8476,
+      "eval_samples_per_second": 73.977,
+      "eval_steps_per_second": 1.167,
+      "step": 6000
+    },
+    {
+      "epoch": 1.7120404153802975,
+      "grad_norm": 0.7958677411079407,
+      "learning_rate": 7.306804715274747e-05,
+      "loss": 3.5277,
+      "step": 6100
+    },
+    {
+      "epoch": 1.7401066516980073,
+      "grad_norm": 0.8098702430725098,
+      "learning_rate": 7.244433356202833e-05,
+      "loss": 3.5095,
+      "step": 6200
+    },
+    {
+      "epoch": 1.768172888015717,
+      "grad_norm": 0.8205238580703735,
+      "learning_rate": 7.182061997130918e-05,
+      "loss": 3.4993,
+      "step": 6300
+    },
+    {
+      "epoch": 1.796239124333427,
+      "grad_norm": 0.8508559465408325,
+      "learning_rate": 7.119690638059004e-05,
+      "loss": 3.4999,
+      "step": 6400
+    },
+    {
+      "epoch": 1.8243053606511368,
+      "grad_norm": 0.7979349493980408,
+      "learning_rate": 7.057319278987089e-05,
+      "loss": 3.4926,
+      "step": 6500
+    },
+    {
+      "epoch": 1.8243053606511368,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.7347623109817505,
+      "eval_perplexity": 5.667580522044148,
+      "eval_runtime": 30.8642,
+      "eval_samples_per_second": 73.937,
+      "eval_steps_per_second": 1.166,
+      "step": 6500
+    },
+    {
+      "epoch": 1.8523715969688466,
+      "grad_norm": 0.7983048558235168,
+      "learning_rate": 6.994947919915176e-05,
+      "loss": 3.4827,
+      "step": 6600
+    },
+    {
+      "epoch": 1.8804378332865563,
+      "grad_norm": 0.8478196263313293,
+      "learning_rate": 6.932576560843261e-05,
+      "loss": 3.4791,
+      "step": 6700
+    },
+    {
+      "epoch": 1.908504069604266,
+      "grad_norm": 0.812839925289154,
+      "learning_rate": 6.870205201771347e-05,
+      "loss": 3.4678,
+      "step": 6800
+    },
+    {
+      "epoch": 1.9365703059219759,
+      "grad_norm": 0.760836660861969,
+      "learning_rate": 6.807833842699432e-05,
+      "loss": 3.4628,
+      "step": 6900
+    },
+    {
+      "epoch": 1.9646365422396856,
+      "grad_norm": 0.8636198043823242,
+      "learning_rate": 6.745462483627518e-05,
+      "loss": 3.464,
+      "step": 7000
+    },
+    {
+      "epoch": 1.9646365422396856,
+      "eval_accuracy": 0.007337265522227044,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.7166798114776611,
+      "eval_perplexity": 5.566017523877059,
+      "eval_runtime": 30.9572,
+      "eval_samples_per_second": 73.715,
+      "eval_steps_per_second": 1.163,
+      "step": 7000
+    },
+    {
+      "epoch": 1.9927027785573954,
+      "grad_norm": 0.8247271776199341,
+      "learning_rate": 6.683091124555604e-05,
+      "loss": 3.4484,
+      "step": 7100
+    },
+    {
+      "epoch": 2.020769014875105,
+      "grad_norm": 0.7949260473251343,
+      "learning_rate": 6.62071976548369e-05,
+      "loss": 3.3852,
+      "step": 7200
+    },
+    {
+      "epoch": 2.048835251192815,
+      "grad_norm": 0.7939648032188416,
+      "learning_rate": 6.558348406411777e-05,
+      "loss": 3.3462,
+      "step": 7300
+    },
+    {
+      "epoch": 2.0769014875105247,
+      "grad_norm": 0.811790943145752,
+      "learning_rate": 6.495977047339862e-05,
+      "loss": 3.3487,
+      "step": 7400
+    },
+    {
+      "epoch": 2.1049677238282345,
+      "grad_norm": 0.8714494705200195,
+      "learning_rate": 6.433605688267948e-05,
+      "loss": 3.3432,
+      "step": 7500
+    },
+    {
+      "epoch": 2.1049677238282345,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.7015873193740845,
+      "eval_perplexity": 5.482643194156826,
+      "eval_runtime": 30.8028,
+      "eval_samples_per_second": 74.084,
+      "eval_steps_per_second": 1.169,
+      "step": 7500
+    },
+    {
+      "epoch": 2.1330339601459443,
+      "grad_norm": 0.8343294262886047,
+      "learning_rate": 6.371234329196033e-05,
+      "loss": 3.3477,
+      "step": 7600
+    },
+    {
+      "epoch": 2.161100196463654,
+      "grad_norm": 0.8184479475021362,
+      "learning_rate": 6.30886297012412e-05,
+      "loss": 3.3313,
+      "step": 7700
+    },
+    {
+      "epoch": 2.189166432781364,
+      "grad_norm": 0.8070995807647705,
+      "learning_rate": 6.246491611052205e-05,
+      "loss": 3.3352,
+      "step": 7800
+    },
+    {
+      "epoch": 2.217232669099074,
+      "grad_norm": 0.8009788393974304,
+      "learning_rate": 6.184120251980291e-05,
+      "loss": 3.3306,
+      "step": 7900
+    },
+    {
+      "epoch": 2.245298905416784,
+      "grad_norm": 0.8022483587265015,
+      "learning_rate": 6.121748892908376e-05,
+      "loss": 3.3215,
+      "step": 8000
+    },
+    {
+      "epoch": 2.245298905416784,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.688325047492981,
+      "eval_perplexity": 5.410410930178965,
+      "eval_runtime": 30.8326,
+      "eval_samples_per_second": 74.013,
+      "eval_steps_per_second": 1.168,
+      "step": 8000
+    },
+    {
+      "epoch": 2.2733651417344936,
+      "grad_norm": 0.8439472317695618,
+      "learning_rate": 6.0593775338364625e-05,
+      "loss": 3.3238,
+      "step": 8100
+    },
+    {
+      "epoch": 2.3014313780522033,
+      "grad_norm": 0.8445388674736023,
+      "learning_rate": 5.9970061747645486e-05,
+      "loss": 3.3158,
+      "step": 8200
+    },
+    {
+      "epoch": 2.329497614369913,
+      "grad_norm": 0.8093782663345337,
+      "learning_rate": 5.934634815692635e-05,
+      "loss": 3.3138,
+      "step": 8300
+    },
+    {
+      "epoch": 2.357563850687623,
+      "grad_norm": 0.8225380778312683,
+      "learning_rate": 5.87226345662072e-05,
+      "loss": 3.3111,
+      "step": 8400
+    },
+    {
+      "epoch": 2.3856300870053326,
+      "grad_norm": 0.8095957040786743,
+      "learning_rate": 5.809892097548806e-05,
+      "loss": 3.3122,
+      "step": 8500
+    },
+    {
+      "epoch": 2.3856300870053326,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.676782488822937,
+      "eval_perplexity": 5.348319978183154,
+      "eval_runtime": 30.8301,
+      "eval_samples_per_second": 74.018,
+      "eval_steps_per_second": 1.168,
+      "step": 8500
+    },
+    {
+      "epoch": 2.4136963233230424,
+      "grad_norm": 0.8112284541130066,
+      "learning_rate": 5.747520738476891e-05,
+      "loss": 3.2971,
+      "step": 8600
+    },
+    {
+      "epoch": 2.441762559640752,
+      "grad_norm": 0.7994459867477417,
+      "learning_rate": 5.685149379404977e-05,
+      "loss": 3.2922,
+      "step": 8700
+    },
+    {
+      "epoch": 2.469828795958462,
+      "grad_norm": 0.8026987910270691,
+      "learning_rate": 5.622778020333064e-05,
+      "loss": 3.2983,
+      "step": 8800
+    },
+    {
+      "epoch": 2.4978950322761717,
+      "grad_norm": 0.8162180781364441,
+      "learning_rate": 5.5604066612611496e-05,
+      "loss": 3.2879,
+      "step": 8900
+    },
+    {
+      "epoch": 2.5259612685938815,
+      "grad_norm": 0.7989441156387329,
+      "learning_rate": 5.498035302189235e-05,
+      "loss": 3.2836,
+      "step": 9000
+    },
+    {
+      "epoch": 2.5259612685938815,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6650645732879639,
+      "eval_perplexity": 5.286014574081701,
+      "eval_runtime": 30.7972,
+      "eval_samples_per_second": 74.098,
+      "eval_steps_per_second": 1.169,
+      "step": 9000
+    },
+    {
+      "epoch": 2.5540275049115913,
+      "grad_norm": 0.7936413288116455,
+      "learning_rate": 5.4356639431173205e-05,
+      "loss": 3.2837,
+      "step": 9100
+    },
+    {
+      "epoch": 2.582093741229301,
+      "grad_norm": 0.8317012190818787,
+      "learning_rate": 5.373292584045406e-05,
+      "loss": 3.2767,
+      "step": 9200
+    },
+    {
+      "epoch": 2.610159977547011,
+      "grad_norm": 0.8149669170379639,
+      "learning_rate": 5.310921224973493e-05,
+      "loss": 3.2627,
+      "step": 9300
+    },
+    {
+      "epoch": 2.638226213864721,
+      "grad_norm": 0.8095958828926086,
+      "learning_rate": 5.248549865901579e-05,
+      "loss": 3.2753,
+      "step": 9400
+    },
+    {
+      "epoch": 2.6662924501824303,
+      "grad_norm": 0.8062021136283875,
+      "learning_rate": 5.1861785068296644e-05,
+      "loss": 3.2582,
+      "step": 9500
+    },
+    {
+      "epoch": 2.6662924501824303,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6540542840957642,
+      "eval_perplexity": 5.22813325470726,
+      "eval_runtime": 30.8763,
+      "eval_samples_per_second": 73.908,
+      "eval_steps_per_second": 1.166,
+      "step": 9500
+    },
+    {
+      "epoch": 2.6943586865001405,
+      "grad_norm": 0.8398740291595459,
+      "learning_rate": 5.12380714775775e-05,
+      "loss": 3.258,
+      "step": 9600
+    },
+    {
+      "epoch": 2.72242492281785,
+      "grad_norm": 0.7982565760612488,
+      "learning_rate": 5.061435788685835e-05,
+      "loss": 3.2541,
+      "step": 9700
+    },
+    {
+      "epoch": 2.75049115913556,
+      "grad_norm": 0.8176411390304565,
+      "learning_rate": 4.9990644296139214e-05,
+      "loss": 3.2556,
+      "step": 9800
+    },
+    {
+      "epoch": 2.77855739545327,
+      "grad_norm": 0.7924163937568665,
+      "learning_rate": 4.9366930705420076e-05,
+      "loss": 3.2548,
+      "step": 9900
+    },
+    {
+      "epoch": 2.8066236317709796,
+      "grad_norm": 0.8106917142868042,
+      "learning_rate": 4.874321711470094e-05,
+      "loss": 3.2387,
+      "step": 10000
+    },
+    {
+      "epoch": 2.8066236317709796,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6433695554733276,
+      "eval_perplexity": 5.172569440429115,
+      "eval_runtime": 30.8104,
+      "eval_samples_per_second": 74.066,
+      "eval_steps_per_second": 1.168,
+      "step": 10000
+    },
+    {
+      "epoch": 2.8346898680886894,
+      "grad_norm": 0.8256287574768066,
+      "learning_rate": 4.811950352398179e-05,
+      "loss": 3.2405,
+      "step": 10100
+    },
+    {
+      "epoch": 2.862756104406399,
+      "grad_norm": 0.802058756351471,
+      "learning_rate": 4.7495789933262646e-05,
+      "loss": 3.2469,
+      "step": 10200
+    },
+    {
+      "epoch": 2.890822340724109,
+      "grad_norm": 0.816027820110321,
+      "learning_rate": 4.687207634254351e-05,
+      "loss": 3.2417,
+      "step": 10300
+    },
+    {
+      "epoch": 2.9188885770418187,
+      "grad_norm": 0.8111074566841125,
+      "learning_rate": 4.624836275182436e-05,
+      "loss": 3.2294,
+      "step": 10400
+    },
+    {
+      "epoch": 2.9469548133595285,
+      "grad_norm": 0.8160965442657471,
+      "learning_rate": 4.5624649161105224e-05,
+      "loss": 3.223,
+      "step": 10500
+    },
+    {
+      "epoch": 2.9469548133595285,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6337802410125732,
+      "eval_perplexity": 5.1232051088182144,
+      "eval_runtime": 30.8817,
+      "eval_samples_per_second": 73.895,
+      "eval_steps_per_second": 1.166,
+      "step": 10500
+    },
+    {
+      "epoch": 2.9750210496772382,
+      "grad_norm": 0.7960518598556519,
+      "learning_rate": 4.5000935570386085e-05,
+      "loss": 3.2314,
+      "step": 10600
+    },
+    {
+      "epoch": 3.003087285994948,
+      "grad_norm": 0.8226578235626221,
+      "learning_rate": 4.437722197966694e-05,
+      "loss": 3.2255,
+      "step": 10700
+    },
+    {
+      "epoch": 3.0311535223126578,
+      "grad_norm": 0.8036494255065918,
+      "learning_rate": 4.3753508388947794e-05,
+      "loss": 3.1286,
+      "step": 10800
+    },
+    {
+      "epoch": 3.0592197586303675,
+      "grad_norm": 0.8422260880470276,
+      "learning_rate": 4.3129794798228656e-05,
+      "loss": 3.1179,
+      "step": 10900
+    },
+    {
+      "epoch": 3.0872859949480773,
+      "grad_norm": 0.8350833058357239,
+      "learning_rate": 4.250608120750951e-05,
+      "loss": 3.126,
+      "step": 11000
+    },
+    {
+      "epoch": 3.0872859949480773,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6289175748825073,
+      "eval_perplexity": 5.098353145228774,
+      "eval_runtime": 30.868,
+      "eval_samples_per_second": 73.928,
+      "eval_steps_per_second": 1.166,
+      "step": 11000
+    },
+    {
+      "epoch": 3.115352231265787,
+      "grad_norm": 0.8042276501655579,
+      "learning_rate": 4.188236761679037e-05,
+      "loss": 3.1171,
+      "step": 11100
+    },
+    {
+      "epoch": 3.143418467583497,
+      "grad_norm": 0.8350549340248108,
+      "learning_rate": 4.125865402607123e-05,
+      "loss": 3.122,
+      "step": 11200
+    },
+    {
+      "epoch": 3.171484703901207,
+      "grad_norm": 0.9750901460647583,
+      "learning_rate": 4.063494043535209e-05,
+      "loss": 3.1236,
+      "step": 11300
+    },
+    {
+      "epoch": 3.199550940218917,
+      "grad_norm": 0.8534584045410156,
+      "learning_rate": 4.001122684463295e-05,
+      "loss": 3.1242,
+      "step": 11400
+    },
+    {
+      "epoch": 3.2276171765366266,
+      "grad_norm": 0.8372871279716492,
+      "learning_rate": 3.9387513253913804e-05,
+      "loss": 3.1149,
+      "step": 11500
+    },
+    {
+      "epoch": 3.2276171765366266,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6211658716201782,
+      "eval_perplexity": 5.058985006716426,
+      "eval_runtime": 30.8572,
+      "eval_samples_per_second": 73.953,
+      "eval_steps_per_second": 1.167,
+      "step": 11500
+    },
+    {
+      "epoch": 3.2556834128543364,
+      "grad_norm": 0.8385348916053772,
+      "learning_rate": 3.876379966319466e-05,
+      "loss": 3.1165,
+      "step": 11600
+    },
+    {
+      "epoch": 3.283749649172046,
+      "grad_norm": 0.8371408581733704,
+      "learning_rate": 3.814008607247552e-05,
+      "loss": 3.1205,
+      "step": 11700
+    },
+    {
+      "epoch": 3.311815885489756,
+      "grad_norm": 1.0450332164764404,
+      "learning_rate": 3.751637248175638e-05,
+      "loss": 3.1143,
+      "step": 11800
+    },
+    {
+      "epoch": 3.3398821218074657,
+      "grad_norm": 0.8579763174057007,
+      "learning_rate": 3.6892658891037236e-05,
+      "loss": 3.1081,
+      "step": 11900
+    },
+    {
+      "epoch": 3.3679483581251755,
+      "grad_norm": 0.8332571983337402,
+      "learning_rate": 3.62689453003181e-05,
+      "loss": 3.1048,
+      "step": 12000
+    },
+    {
+      "epoch": 3.3679483581251755,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6149429082870483,
+      "eval_perplexity": 5.027600880928623,
+      "eval_runtime": 30.7824,
+      "eval_samples_per_second": 74.133,
+      "eval_steps_per_second": 1.169,
+      "step": 12000
+    },
+    {
+      "epoch": 3.3960145944428852,
+      "grad_norm": 0.8373920321464539,
+      "learning_rate": 3.564523170959895e-05,
+      "loss": 3.1223,
+      "step": 12100
+    },
+    {
+      "epoch": 3.424080830760595,
+      "grad_norm": 0.8379535675048828,
+      "learning_rate": 3.502151811887981e-05,
+      "loss": 3.1178,
+      "step": 12200
+    },
+    {
+      "epoch": 3.4521470670783048,
+      "grad_norm": 0.8385716080665588,
+      "learning_rate": 3.439780452816067e-05,
+      "loss": 3.1028,
+      "step": 12300
+    },
+    {
+      "epoch": 3.4802133033960145,
+      "grad_norm": 0.8330587148666382,
+      "learning_rate": 3.377409093744153e-05,
+      "loss": 3.1069,
+      "step": 12400
+    },
+    {
+      "epoch": 3.5082795397137243,
+      "grad_norm": 0.8396536111831665,
+      "learning_rate": 3.315037734672239e-05,
+      "loss": 3.0966,
+      "step": 12500
+    },
+    {
+      "epoch": 3.5082795397137243,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6073817014694214,
+      "eval_perplexity": 4.989729507944433,
+      "eval_runtime": 30.8556,
+      "eval_samples_per_second": 73.957,
+      "eval_steps_per_second": 1.167,
+      "step": 12500
+    },
+    {
+      "epoch": 3.536345776031434,
+      "grad_norm": 0.8449435234069824,
+      "learning_rate": 3.2526663756003245e-05,
+      "loss": 3.1004,
+      "step": 12600
+    },
+    {
+      "epoch": 3.564412012349144,
+      "grad_norm": 0.8464480042457581,
+      "learning_rate": 3.19029501652841e-05,
+      "loss": 3.1037,
+      "step": 12700
+    },
+    {
+      "epoch": 3.5924782486668536,
+      "grad_norm": 0.8337912559509277,
+      "learning_rate": 3.127923657456496e-05,
+      "loss": 3.0935,
+      "step": 12800
+    },
+    {
+      "epoch": 3.6205444849845634,
+      "grad_norm": 0.8544827103614807,
+      "learning_rate": 3.065552298384582e-05,
+      "loss": 3.0967,
+      "step": 12900
+    },
+    {
+      "epoch": 3.6486107213022736,
+      "grad_norm": 0.8739194869995117,
+      "learning_rate": 3.0031809393126674e-05,
+      "loss": 3.0904,
+      "step": 13000
+    },
+    {
+      "epoch": 3.6486107213022736,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.6013461351394653,
+      "eval_perplexity": 4.959704365047878,
+      "eval_runtime": 30.8424,
+      "eval_samples_per_second": 73.989,
+      "eval_steps_per_second": 1.167,
+      "step": 13000
+    },
+    {
+      "epoch": 3.676676957619983,
+      "grad_norm": 1.3021411895751953,
+      "learning_rate": 2.940809580240754e-05,
+      "loss": 3.0893,
+      "step": 13100
+    },
+    {
+      "epoch": 3.704743193937693,
+      "grad_norm": 0.8261712193489075,
+      "learning_rate": 2.8784382211688393e-05,
+      "loss": 3.0913,
+      "step": 13200
+    },
+    {
+      "epoch": 3.732809430255403,
+      "grad_norm": 0.8515812754631042,
+      "learning_rate": 2.8160668620969254e-05,
+      "loss": 3.0891,
+      "step": 13300
+    },
+    {
+      "epoch": 3.7608756665731127,
+      "grad_norm": 0.8673535585403442,
+      "learning_rate": 2.7536955030250112e-05,
+      "loss": 3.08,
+      "step": 13400
+    },
+    {
+      "epoch": 3.7889419028908224,
+      "grad_norm": 0.8489654660224915,
+      "learning_rate": 2.6913241439530967e-05,
+      "loss": 3.0979,
+      "step": 13500
+    },
+    {
+      "epoch": 3.7889419028908224,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.595861554145813,
+      "eval_perplexity": 4.932576924077616,
+      "eval_runtime": 30.8403,
+      "eval_samples_per_second": 73.994,
+      "eval_steps_per_second": 1.167,
+      "step": 13500
+    },
+    {
+      "epoch": 3.817008139208532,
+      "grad_norm": 0.8460671901702881,
+      "learning_rate": 2.628952784881183e-05,
+      "loss": 3.0687,
+      "step": 13600
+    },
+    {
+      "epoch": 3.845074375526242,
+      "grad_norm": 0.8813645839691162,
+      "learning_rate": 2.5665814258092686e-05,
+      "loss": 3.0696,
+      "step": 13700
+    },
+    {
+      "epoch": 3.8731406118439518,
+      "grad_norm": 0.8696038126945496,
+      "learning_rate": 2.504210066737354e-05,
+      "loss": 3.07,
+      "step": 13800
+    },
+    {
+      "epoch": 3.9012068481616615,
+      "grad_norm": 0.8538317680358887,
+      "learning_rate": 2.44183870766544e-05,
+      "loss": 3.0845,
+      "step": 13900
+    },
+    {
+      "epoch": 3.9292730844793713,
+      "grad_norm": 0.8570790886878967,
+      "learning_rate": 2.379467348593526e-05,
+      "loss": 3.0772,
+      "step": 14000
+    },
+    {
+      "epoch": 3.9292730844793713,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.5907403230667114,
+      "eval_perplexity": 4.907380630922354,
+      "eval_runtime": 30.8034,
+      "eval_samples_per_second": 74.083,
+      "eval_steps_per_second": 1.169,
+      "step": 14000
+    },
+    {
+      "epoch": 3.957339320797081,
+      "grad_norm": 0.8513247966766357,
+      "learning_rate": 2.317095989521612e-05,
+      "loss": 3.0714,
+      "step": 14100
+    },
+    {
+      "epoch": 3.985405557114791,
+      "grad_norm": 0.8460662961006165,
+      "learning_rate": 2.2547246304496976e-05,
+      "loss": 3.0662,
+      "step": 14200
+    },
+    {
+      "epoch": 4.013471793432501,
+      "grad_norm": 0.8479135632514954,
+      "learning_rate": 2.1923532713777834e-05,
+      "loss": 3.0293,
+      "step": 14300
+    },
+    {
+      "epoch": 4.04153802975021,
+      "grad_norm": 0.8859968185424805,
+      "learning_rate": 2.1299819123058692e-05,
+      "loss": 2.9782,
+      "step": 14400
+    },
+    {
+      "epoch": 4.069604266067921,
+      "grad_norm": 0.8683611750602722,
+      "learning_rate": 2.067610553233955e-05,
+      "loss": 2.9795,
+      "step": 14500
+    },
+    {
+      "epoch": 4.069604266067921,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.5887154340744019,
+      "eval_perplexity": 4.897453783675403,
+      "eval_runtime": 30.7941,
+      "eval_samples_per_second": 74.105,
+      "eval_steps_per_second": 1.169,
+      "step": 14500
+    },
+    {
+      "epoch": 4.09767050238563,
+      "grad_norm": 0.8947647213935852,
+      "learning_rate": 2.005239194162041e-05,
+      "loss": 2.9837,
+      "step": 14600
+    },
+    {
+      "epoch": 4.12573673870334,
+      "grad_norm": 0.8813712000846863,
+      "learning_rate": 1.9428678350901266e-05,
+      "loss": 2.9733,
+      "step": 14700
+    },
+    {
+      "epoch": 4.1538029750210494,
+      "grad_norm": 0.8821486234664917,
+      "learning_rate": 1.8804964760182124e-05,
+      "loss": 2.9878,
+      "step": 14800
+    },
+    {
+      "epoch": 4.18186921133876,
+      "grad_norm": 0.9034985899925232,
+      "learning_rate": 1.8181251169462986e-05,
+      "loss": 2.9819,
+      "step": 14900
+    },
+    {
+      "epoch": 4.209935447656469,
+      "grad_norm": 0.9136458039283752,
+      "learning_rate": 1.755753757874384e-05,
+      "loss": 2.9807,
+      "step": 15000
+    },
+    {
+      "epoch": 4.209935447656469,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.5847208499908447,
+      "eval_perplexity": 4.877929514370786,
+      "eval_runtime": 30.8617,
+      "eval_samples_per_second": 73.943,
+      "eval_steps_per_second": 1.166,
+      "step": 15000
+    },
+    {
+      "epoch": 4.238001683974179,
+      "grad_norm": 0.900557816028595,
+      "learning_rate": 1.69338239880247e-05,
+      "loss": 2.9784,
+      "step": 15100
+    },
+    {
+      "epoch": 4.2660679202918885,
+      "grad_norm": 0.8973119258880615,
+      "learning_rate": 1.631011039730556e-05,
+      "loss": 2.9765,
+      "step": 15200
+    },
+    {
+      "epoch": 4.294134156609599,
+      "grad_norm": 0.913901686668396,
+      "learning_rate": 1.5686396806586418e-05,
+      "loss": 2.9763,
+      "step": 15300
+    },
+    {
+      "epoch": 4.322200392927308,
+      "grad_norm": 0.8976321816444397,
+      "learning_rate": 1.5062683215867274e-05,
+      "loss": 2.9718,
+      "step": 15400
+    },
+    {
+      "epoch": 4.350266629245018,
+      "grad_norm": 0.904545247554779,
+      "learning_rate": 1.4438969625148132e-05,
+      "loss": 2.976,
+      "step": 15500
+    },
+    {
+      "epoch": 4.350266629245018,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.5817325115203857,
+      "eval_perplexity": 4.86337436862944,
+      "eval_runtime": 30.8917,
+      "eval_samples_per_second": 73.871,
+      "eval_steps_per_second": 1.165,
+      "step": 15500
+    },
+    {
+      "epoch": 4.378332865562728,
+      "grad_norm": 0.9036585092544556,
+      "learning_rate": 1.3815256034428992e-05,
+      "loss": 2.9729,
+      "step": 15600
+    },
+    {
+      "epoch": 4.406399101880438,
+      "grad_norm": 0.8927172422409058,
+      "learning_rate": 1.3191542443709848e-05,
+      "loss": 2.9622,
+      "step": 15700
+    },
+    {
+      "epoch": 4.434465338198148,
+      "grad_norm": 0.8881782293319702,
+      "learning_rate": 1.2567828852990706e-05,
+      "loss": 2.974,
+      "step": 15800
+    },
+    {
+      "epoch": 4.462531574515857,
+      "grad_norm": 0.8996316194534302,
+      "learning_rate": 1.1944115262271566e-05,
+      "loss": 2.9673,
+      "step": 15900
+    },
+    {
+      "epoch": 4.490597810833568,
+      "grad_norm": 0.9018428921699524,
+      "learning_rate": 1.1320401671552424e-05,
+      "loss": 2.9795,
+      "step": 16000
+    },
+    {
+      "epoch": 4.490597810833568,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.5778179168701172,
+      "eval_perplexity": 4.844373444066827,
+      "eval_runtime": 30.8539,
+      "eval_samples_per_second": 73.961,
+      "eval_steps_per_second": 1.167,
+      "step": 16000
+    },
+    {
+      "epoch": 4.518664047151277,
+      "grad_norm": 0.8891787528991699,
+      "learning_rate": 1.0696688080833282e-05,
+      "loss": 2.9647,
+      "step": 16100
+    },
+    {
+      "epoch": 4.546730283468987,
+      "grad_norm": 0.9130581021308899,
+      "learning_rate": 1.007297449011414e-05,
+      "loss": 2.9789,
+      "step": 16200
+    },
+    {
+      "epoch": 4.574796519786696,
+      "grad_norm": 0.8879275918006897,
+      "learning_rate": 9.449260899395e-06,
+      "loss": 2.9773,
+      "step": 16300
+    },
+    {
+      "epoch": 4.602862756104407,
+      "grad_norm": 0.9074363112449646,
+      "learning_rate": 8.825547308675856e-06,
+      "loss": 2.9711,
+      "step": 16400
+    },
+    {
+      "epoch": 4.630928992422116,
+      "grad_norm": 0.8947946429252625,
+      "learning_rate": 8.201833717956715e-06,
+      "loss": 2.9594,
+      "step": 16500
+    },
+    {
+      "epoch": 4.630928992422116,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.5754330158233643,
+      "eval_perplexity": 4.832833858623437,
+      "eval_runtime": 30.8924,
+      "eval_samples_per_second": 73.869,
+      "eval_steps_per_second": 1.165,
+      "step": 16500
+    },
+    {
+      "epoch": 4.658995228739826,
+      "grad_norm": 0.9059449434280396,
+      "learning_rate": 7.578120127237573e-06,
+      "loss": 2.9653,
+      "step": 16600
+    },
+    {
+      "epoch": 4.6870614650575355,
+      "grad_norm": 0.9055649638175964,
+      "learning_rate": 6.954406536518431e-06,
+      "loss": 2.9653,
+      "step": 16700
+    },
+    {
+      "epoch": 4.715127701375246,
+      "grad_norm": 0.9104346036911011,
+      "learning_rate": 6.3306929457992894e-06,
+      "loss": 2.9689,
+      "step": 16800
+    },
+    {
+      "epoch": 4.743193937692955,
+      "grad_norm": 0.8993477821350098,
+      "learning_rate": 5.7069793550801474e-06,
+      "loss": 2.9637,
+      "step": 16900
+    },
+    {
+      "epoch": 4.771260174010665,
+      "grad_norm": 0.9155908226966858,
+      "learning_rate": 5.083265764361006e-06,
+      "loss": 2.9698,
+      "step": 17000
+    },
+    {
+      "epoch": 4.771260174010665,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.5731316804885864,
+      "eval_perplexity": 4.821724675178664,
+      "eval_runtime": 30.8071,
+      "eval_samples_per_second": 74.074,
+      "eval_steps_per_second": 1.169,
+      "step": 17000
+    },
+    {
+      "epoch": 4.799326410328375,
+      "grad_norm": 0.9122122526168823,
+      "learning_rate": 4.4595521736418634e-06,
+      "loss": 2.9619,
+      "step": 17100
+    },
+    {
+      "epoch": 4.827392646646085,
+      "grad_norm": 0.9053720831871033,
+      "learning_rate": 3.8358385829227214e-06,
+      "loss": 2.9557,
+      "step": 17200
+    },
+    {
+      "epoch": 4.855458882963795,
+      "grad_norm": 0.9079867601394653,
+      "learning_rate": 3.2121249922035803e-06,
+      "loss": 2.9681,
+      "step": 17300
+    },
+    {
+      "epoch": 4.883525119281504,
+      "grad_norm": 0.911987841129303,
+      "learning_rate": 2.5884114014844383e-06,
+      "loss": 2.9475,
+      "step": 17400
+    },
+    {
+      "epoch": 4.911591355599214,
+      "grad_norm": 0.9014911651611328,
+      "learning_rate": 1.9646978107652967e-06,
+      "loss": 2.9604,
+      "step": 17500
+    },
+    {
+      "epoch": 4.911591355599214,
+      "eval_accuracy": 0.00733640796431187,
+      "eval_bleu": 1.0,
+      "eval_loss": 1.5712791681289673,
+      "eval_perplexity": 4.8128006391188904,
+      "eval_runtime": 30.8544,
+      "eval_samples_per_second": 73.96,
+      "eval_steps_per_second": 1.167,
+      "step": 17500
+    },
+    {
+      "epoch": 4.939657591916924,
+      "grad_norm": 0.9085986614227295,
+      "learning_rate": 1.340984220046155e-06,
+      "loss": 2.9556,
+      "step": 17600
+    },
+    {
+      "epoch": 4.967723828234634,
+      "grad_norm": 0.8967222571372986,
+      "learning_rate": 7.172706293270131e-07,
+      "loss": 2.972,
+      "step": 17700
+    },
+    {
+      "epoch": 4.995790064552343,
+      "grad_norm": 0.8958012461662292,
+      "learning_rate": 9.355703860787128e-08,
+      "loss": 2.9616,
+      "step": 17800
+    },
     {
       "epoch": 5.0,
-      "step": 5,
-      "total_flos": 129891126435840.0,
-      "train_loss": 19.52422180175781,
-      "train_runtime": 23.5998,
-      "train_samples_per_second": 2.966,
-      "train_steps_per_second": 0.212
+      "step": 17815,
+      "total_flos": 2.1154254410092954e+18,
+      "train_loss": 3.7582801998715505,
+      "train_runtime": 52733.3783,
+      "train_samples_per_second": 21.619,
+      "train_steps_per_second": 0.338
     }
   ],
   "logging_steps": 100,
-  "max_steps": 5,
+  "max_steps": 17815,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 5,
   "save_steps": 500,
@@ -44,7 +1675,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 129891126435840.0,
+  "total_flos": 2.1154254410092954e+18,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15a9b27cccf8738a58eddc8b80e6f2f2f5a1de46826c0486b88114a34ebaa8bc
-size 5496
+oid sha256:06b88e540f475a3df78ab599583a624266c4e626b709d17699358ae255edf5c7
+size 5560