Training in progress, epoch 2

all_results.json

@@ -1,26 +1,26 @@
 {
-    "epoch": 14.0,
-    "eval_gen_len": 16.38,
-    "eval_loss": 2.551440477371216,
-    "eval_rouge-1": 64.0458,
-    "eval_rouge-2": 54.1763,
-    "eval_rouge-l": 63.7781,
-    "eval_runtime": 84.1655,
+    "epoch": 20.0,
+    "eval_gen_len": 27.2356,
+    "eval_loss": 2.2613940238952637,
+    "eval_rouge-1": 72.7529,
+    "eval_rouge-2": 64.3941,
+    "eval_rouge-l": 71.5976,
+    "eval_runtime": 129.9628,
     "eval_samples": 450,
-    "eval_samples_per_second": 5.347,
-    "eval_steps_per_second": 0.677,
-    "predict_gen_len": 15.8561,
-    "predict_loss": 2.650156259536743,
-    "predict_rouge-1": 64.0995,
-    "predict_rouge-2": 53.7587,
-    "predict_rouge-l": 63.5353,
-    "predict_runtime": 74.6719,
+    "eval_samples_per_second": 3.463,
+    "eval_steps_per_second": 0.439,
+    "predict_gen_len": 26.4604,
+    "predict_loss": 2.305396795272827,
+    "predict_rouge-1": 73.3028,
+    "predict_rouge-2": 65.6876,
+    "predict_rouge-l": 72.5101,
+    "predict_runtime": 115.003,
     "predict_samples": 417,
-    "predict_samples_per_second": 5.584,
-    "predict_steps_per_second": 0.71,
-    "train_loss": 2.2515651293738874,
-    "train_runtime": 5205.8134,
+    "predict_samples_per_second": 3.626,
+    "predict_steps_per_second": 0.461,
+    "train_loss": 2.047671499282029,
+    "train_runtime": 9522.6649,
     "train_samples": 5133,
-    "train_samples_per_second": 49.301,
-    "train_steps_per_second": 6.166
+    "train_samples_per_second": 26.951,
+    "train_steps_per_second": 3.371
 }

eval_results.json

@@ -1,12 +1,12 @@
 {
-    "epoch": 14.0,
-    "eval_gen_len": 16.38,
-    "eval_loss": 2.551440477371216,
-    "eval_rouge-1": 64.0458,
-    "eval_rouge-2": 54.1763,
-    "eval_rouge-l": 63.7781,
-    "eval_runtime": 84.1655,
+    "epoch": 20.0,
+    "eval_gen_len": 27.2356,
+    "eval_loss": 2.2613940238952637,
+    "eval_rouge-1": 72.7529,
+    "eval_rouge-2": 64.3941,
+    "eval_rouge-l": 71.5976,
+    "eval_runtime": 129.9628,
     "eval_samples": 450,
-    "eval_samples_per_second": 5.347,
-    "eval_steps_per_second": 0.677
+    "eval_samples_per_second": 3.463,
+    "eval_steps_per_second": 0.439
 }

predict_results.json

@@ -1,11 +1,11 @@
 {
-    "predict_gen_len": 15.8561,
-    "predict_loss": 2.650156259536743,
-    "predict_rouge-1": 64.0995,
-    "predict_rouge-2": 53.7587,
-    "predict_rouge-l": 63.5353,
-    "predict_runtime": 74.6719,
+    "predict_gen_len": 26.4604,
+    "predict_loss": 2.305396795272827,
+    "predict_rouge-1": 73.3028,
+    "predict_rouge-2": 65.6876,
+    "predict_rouge-l": 72.5101,
+    "predict_runtime": 115.003,
     "predict_samples": 417,
-    "predict_samples_per_second": 5.584,
-    "predict_steps_per_second": 0.71
+    "predict_samples_per_second": 3.626,
+    "predict_steps_per_second": 0.461
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 14.0,
-    "train_loss": 2.2515651293738874,
-    "train_runtime": 5205.8134,
+    "epoch": 20.0,
+    "train_loss": 2.047671499282029,
+    "train_runtime": 9522.6649,
     "train_samples": 5133,
-    "train_samples_per_second": 49.301,
-    "train_steps_per_second": 6.166
+    "train_samples_per_second": 26.951,
+    "train_steps_per_second": 3.371
 }

trainer_state.json

@@ -1,202 +1,280 @@
 {
-  "best_metric": 2.551440477371216,
-  "best_model_checkpoint": "PEFT/adapters-lib/output/mam/dataset-5100/test/checkpoint-5778",
-  "epoch": 14.0,
+  "best_metric": 2.2613940238952637,
+  "best_model_checkpoint": "PEFT/adapters-lib/output/mam/dataset-5100/test/checkpoint-9630",
+  "epoch": 20.0,
   "eval_steps": 500,
-  "global_step": 8988,
+  "global_step": 12840,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "eval_gen_len": 17.9067,
-      "eval_loss": 3.148862361907959,
-      "eval_rouge-1": 34.9145,
-      "eval_rouge-2": 18.886,
-      "eval_rouge-l": 34.0576,
-      "eval_runtime": 92.7146,
-      "eval_samples_per_second": 4.854,
-      "eval_steps_per_second": 0.615,
+      "eval_gen_len": 27.1578,
+      "eval_loss": 3.0984482765197754,
+      "eval_rouge-1": 35.9221,
+      "eval_rouge-2": 20.801,
+      "eval_rouge-l": 33.0593,
+      "eval_runtime": 132.8327,
+      "eval_samples_per_second": 3.388,
+      "eval_steps_per_second": 0.429,
       "step": 642
     },
     {
       "epoch": 2.0,
-      "eval_gen_len": 20.4022,
-      "eval_loss": 2.9582486152648926,
-      "eval_rouge-1": 36.7329,
-      "eval_rouge-2": 20.2718,
-      "eval_rouge-l": 36.107,
-      "eval_runtime": 105.7605,
-      "eval_samples_per_second": 4.255,
-      "eval_steps_per_second": 0.539,
+      "eval_gen_len": 23.6511,
+      "eval_loss": 2.8489246368408203,
+      "eval_rouge-1": 39.2966,
+      "eval_rouge-2": 22.654,
+      "eval_rouge-l": 37.1498,
+      "eval_runtime": 117.4318,
+      "eval_samples_per_second": 3.832,
+      "eval_steps_per_second": 0.485,
       "step": 1284
     },
     {
       "epoch": 3.0,
-      "eval_gen_len": 17.6689,
-      "eval_loss": 2.790357828140259,
-      "eval_rouge-1": 41.2473,
-      "eval_rouge-2": 24.743,
-      "eval_rouge-l": 40.7266,
-      "eval_runtime": 92.3058,
-      "eval_samples_per_second": 4.875,
-      "eval_steps_per_second": 0.618,
+      "eval_gen_len": 26.7267,
+      "eval_loss": 2.5933380126953125,
+      "eval_rouge-1": 47.1185,
+      "eval_rouge-2": 30.2869,
+      "eval_rouge-l": 44.1491,
+      "eval_runtime": 135.6918,
+      "eval_samples_per_second": 3.316,
+      "eval_steps_per_second": 0.42,
       "step": 1926
     },
     {
       "epoch": 4.0,
-      "eval_gen_len": 16.3289,
-      "eval_loss": 2.692574977874756,
-      "eval_rouge-1": 48.3255,
-      "eval_rouge-2": 31.9378,
-      "eval_rouge-l": 47.3541,
-      "eval_runtime": 81.915,
-      "eval_samples_per_second": 5.493,
-      "eval_steps_per_second": 0.696,
+      "eval_gen_len": 25.22,
+      "eval_loss": 2.476445198059082,
+      "eval_rouge-1": 50.0244,
+      "eval_rouge-2": 34.9212,
+      "eval_rouge-l": 48.057,
+      "eval_runtime": 123.2916,
+      "eval_samples_per_second": 3.65,
+      "eval_steps_per_second": 0.462,
       "step": 2568
     },
     {
       "epoch": 5.0,
-      "eval_gen_len": 16.1467,
-      "eval_loss": 2.6225757598876953,
-      "eval_rouge-1": 52.4513,
-      "eval_rouge-2": 38.1209,
-      "eval_rouge-l": 52.1258,
-      "eval_runtime": 80.1926,
-      "eval_samples_per_second": 5.611,
-      "eval_steps_per_second": 0.711,
+      "eval_gen_len": 26.6533,
+      "eval_loss": 2.412160873413086,
+      "eval_rouge-1": 54.5479,
+      "eval_rouge-2": 40.7397,
+      "eval_rouge-l": 52.5031,
+      "eval_runtime": 131.3008,
+      "eval_samples_per_second": 3.427,
+      "eval_steps_per_second": 0.434,
       "step": 3210
     },
     {
       "epoch": 6.0,
-      "eval_gen_len": 16.6178,
-      "eval_loss": 2.589585542678833,
-      "eval_rouge-1": 54.5309,
-      "eval_rouge-2": 41.7086,
-      "eval_rouge-l": 54.1448,
-      "eval_runtime": 83.352,
-      "eval_samples_per_second": 5.399,
-      "eval_steps_per_second": 0.684,
+      "eval_gen_len": 26.2667,
+      "eval_loss": 2.342738151550293,
+      "eval_rouge-1": 57.6653,
+      "eval_rouge-2": 44.2728,
+      "eval_rouge-l": 55.9363,
+      "eval_runtime": 127.5752,
+      "eval_samples_per_second": 3.527,
+      "eval_steps_per_second": 0.447,
       "step": 3852
     },
     {
       "epoch": 7.0,
-      "eval_gen_len": 16.6844,
-      "eval_loss": 2.613889455795288,
-      "eval_rouge-1": 59.101,
-      "eval_rouge-2": 46.7108,
-      "eval_rouge-l": 58.6428,
-      "eval_runtime": 83.7203,
-      "eval_samples_per_second": 5.375,
-      "eval_steps_per_second": 0.681,
+      "eval_gen_len": 26.1711,
+      "eval_loss": 2.3344807624816895,
+      "eval_rouge-1": 62.4422,
+      "eval_rouge-2": 50.3255,
+      "eval_rouge-l": 60.7065,
+      "eval_runtime": 127.1935,
+      "eval_samples_per_second": 3.538,
+      "eval_steps_per_second": 0.448,
       "step": 4494
     },
     {
       "epoch": 7.79,
       "learning_rate": 8.715112540192926e-05,
-      "loss": 2.6207,
+      "loss": 2.5283,
       "step": 5000
     },
     {
       "epoch": 8.0,
-      "eval_gen_len": 16.5756,
-      "eval_loss": 2.589402675628662,
-      "eval_rouge-1": 63.603,
-      "eval_rouge-2": 52.3404,
-      "eval_rouge-l": 63.3017,
-      "eval_runtime": 85.7704,
-      "eval_samples_per_second": 5.247,
-      "eval_steps_per_second": 0.665,
+      "eval_gen_len": 26.3044,
+      "eval_loss": 2.3082587718963623,
+      "eval_rouge-1": 65.6385,
+      "eval_rouge-2": 55.386,
+      "eval_rouge-l": 64.2792,
+      "eval_runtime": 129.227,
+      "eval_samples_per_second": 3.482,
+      "eval_steps_per_second": 0.441,
       "step": 5136
     },
     {
       "epoch": 9.0,
-      "eval_gen_len": 16.38,
-      "eval_loss": 2.551440477371216,
-      "eval_rouge-1": 64.0458,
-      "eval_rouge-2": 54.1763,
-      "eval_rouge-l": 63.7781,
-      "eval_runtime": 84.9951,
-      "eval_samples_per_second": 5.294,
-      "eval_steps_per_second": 0.671,
+      "eval_gen_len": 26.0844,
+      "eval_loss": 2.288771390914917,
+      "eval_rouge-1": 69.2728,
+      "eval_rouge-2": 59.5819,
+      "eval_rouge-l": 68.0772,
+      "eval_runtime": 127.02,
+      "eval_samples_per_second": 3.543,
+      "eval_steps_per_second": 0.449,
       "step": 5778
     },
     {
       "epoch": 10.0,
-      "eval_gen_len": 16.2444,
-      "eval_loss": 2.5532784461975098,
-      "eval_rouge-1": 65.4167,
-      "eval_rouge-2": 55.446,
-      "eval_rouge-l": 65.0577,
-      "eval_runtime": 80.4855,
-      "eval_samples_per_second": 5.591,
-      "eval_steps_per_second": 0.708,
+      "eval_gen_len": 26.5111,
+      "eval_loss": 2.2961621284484863,
+      "eval_rouge-1": 69.5351,
+      "eval_rouge-2": 60.4728,
+      "eval_rouge-l": 68.5563,
+      "eval_runtime": 128.3845,
+      "eval_samples_per_second": 3.505,
+      "eval_steps_per_second": 0.444,
       "step": 6420
     },
     {
       "epoch": 11.0,
-      "eval_gen_len": 16.5644,
-      "eval_loss": 2.5620651245117188,
-      "eval_rouge-1": 67.0846,
-      "eval_rouge-2": 57.8265,
-      "eval_rouge-l": 66.7532,
-      "eval_runtime": 82.9275,
-      "eval_samples_per_second": 5.426,
-      "eval_steps_per_second": 0.687,
+      "eval_gen_len": 26.4467,
+      "eval_loss": 2.294877052307129,
+      "eval_rouge-1": 71.1387,
+      "eval_rouge-2": 62.2653,
+      "eval_rouge-l": 69.9201,
+      "eval_runtime": 125.5037,
+      "eval_samples_per_second": 3.586,
+      "eval_steps_per_second": 0.454,
       "step": 7062
     },
     {
       "epoch": 12.0,
-      "eval_gen_len": 16.3689,
-      "eval_loss": 2.5536558628082275,
-      "eval_rouge-1": 66.2595,
-      "eval_rouge-2": 56.4953,
-      "eval_rouge-l": 65.7888,
-      "eval_runtime": 82.8266,
-      "eval_samples_per_second": 5.433,
-      "eval_steps_per_second": 0.688,
+      "eval_gen_len": 26.3911,
+      "eval_loss": 2.270343065261841,
+      "eval_rouge-1": 72.6498,
+      "eval_rouge-2": 63.8234,
+      "eval_rouge-l": 71.505,
+      "eval_runtime": 128.4901,
+      "eval_samples_per_second": 3.502,
+      "eval_steps_per_second": 0.444,
       "step": 7704
     },
     {
       "epoch": 13.0,
-      "eval_gen_len": 16.1778,
-      "eval_loss": 2.577864646911621,
-      "eval_rouge-1": 66.3899,
-      "eval_rouge-2": 56.6339,
-      "eval_rouge-l": 66.1145,
-      "eval_runtime": 84.6465,
-      "eval_samples_per_second": 5.316,
-      "eval_steps_per_second": 0.673,
+      "eval_gen_len": 26.3556,
+      "eval_loss": 2.2668042182922363,
+      "eval_rouge-1": 73.2494,
+      "eval_rouge-2": 65.1934,
+      "eval_rouge-l": 72.2586,
+      "eval_runtime": 125.8233,
+      "eval_samples_per_second": 3.576,
+      "eval_steps_per_second": 0.453,
       "step": 8346
     },
     {
       "epoch": 14.0,
-      "eval_gen_len": 16.3289,
-      "eval_loss": 2.5883078575134277,
-      "eval_rouge-1": 67.054,
-      "eval_rouge-2": 58.2501,
-      "eval_rouge-l": 66.8589,
-      "eval_runtime": 82.6488,
-      "eval_samples_per_second": 5.445,
-      "eval_steps_per_second": 0.69,
+      "eval_gen_len": 26.5333,
+      "eval_loss": 2.267202854156494,
+      "eval_rouge-1": 73.2228,
+      "eval_rouge-2": 64.7993,
+      "eval_rouge-l": 71.9902,
+      "eval_runtime": 128.1385,
+      "eval_samples_per_second": 3.512,
+      "eval_steps_per_second": 0.445,
       "step": 8988
     },
     {
-      "epoch": 14.0,
-      "step": 8988,
-      "total_flos": 2.425279776546816e+16,
-      "train_loss": 2.2515651293738874,
-      "train_runtime": 5205.8134,
-      "train_samples_per_second": 49.301,
-      "train_steps_per_second": 6.166
+      "epoch": 15.0,
+      "eval_gen_len": 27.2356,
+      "eval_loss": 2.2613940238952637,
+      "eval_rouge-1": 72.7529,
+      "eval_rouge-2": 64.3941,
+      "eval_rouge-l": 71.5976,
+      "eval_runtime": 129.55,
+      "eval_samples_per_second": 3.474,
+      "eval_steps_per_second": 0.44,
+      "step": 9630
+    },
+    {
+      "epoch": 15.58,
+      "learning_rate": 7.109003215434083e-05,
+      "loss": 1.7733,
+      "step": 10000
+    },
+    {
+      "epoch": 16.0,
+      "eval_gen_len": 26.5311,
+      "eval_loss": 2.2660067081451416,
+      "eval_rouge-1": 73.5002,
+      "eval_rouge-2": 65.0894,
+      "eval_rouge-l": 72.2767,
+      "eval_runtime": 129.1203,
+      "eval_samples_per_second": 3.485,
+      "eval_steps_per_second": 0.441,
+      "step": 10272
+    },
+    {
+      "epoch": 17.0,
+      "eval_gen_len": 26.4089,
+      "eval_loss": 2.2624781131744385,
+      "eval_rouge-1": 73.9016,
+      "eval_rouge-2": 66.1357,
+      "eval_rouge-l": 72.7275,
+      "eval_runtime": 125.6051,
+      "eval_samples_per_second": 3.583,
+      "eval_steps_per_second": 0.454,
+      "step": 10914
+    },
+    {
+      "epoch": 18.0,
+      "eval_gen_len": 26.0111,
+      "eval_loss": 2.2674496173858643,
+      "eval_rouge-1": 74.1365,
+      "eval_rouge-2": 66.7011,
+      "eval_rouge-l": 73.0304,
+      "eval_runtime": 123.8827,
+      "eval_samples_per_second": 3.632,
+      "eval_steps_per_second": 0.46,
+      "step": 11556
+    },
+    {
+      "epoch": 19.0,
+      "eval_gen_len": 26.7356,
+      "eval_loss": 2.264456033706665,
+      "eval_rouge-1": 74.1459,
+      "eval_rouge-2": 66.8989,
+      "eval_rouge-l": 73.1443,
+      "eval_runtime": 128.4694,
+      "eval_samples_per_second": 3.503,
+      "eval_steps_per_second": 0.444,
+      "step": 12198
+    },
+    {
+      "epoch": 20.0,
+      "eval_gen_len": 26.5311,
+      "eval_loss": 2.2706127166748047,
+      "eval_rouge-1": 74.3728,
+      "eval_rouge-2": 66.4207,
+      "eval_rouge-l": 73.2288,
+      "eval_runtime": 125.0151,
+      "eval_samples_per_second": 3.6,
+      "eval_steps_per_second": 0.456,
+      "step": 12840
+    },
+    {
+      "epoch": 20.0,
+      "step": 12840,
+      "total_flos": 3.46468539506688e+16,
+      "train_loss": 2.047671499282029,
+      "train_runtime": 9522.6649,
+      "train_samples_per_second": 26.951,
+      "train_steps_per_second": 3.371
     }
   ],
   "logging_steps": 5000,
   "max_steps": 32100,
   "num_train_epochs": 50,
   "save_steps": 500,
-  "total_flos": 2.425279776546816e+16,
+  "total_flos": 3.46468539506688e+16,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e30373099d2e53d4da1554720e526088b2daf0dd3fe067ef1087c8172275755b
+oid sha256:a95829ca88d043b246dcabe92dff664cfa4ccae9daef7724eeb0be2f31e5b5b5
 size 4792