Model save

README.md

@@ -17,10 +17,10 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 3.1790
 - Accuracy: 0.4217
-- Perplexity: 24.0231
 - Bleu: 0.1309
+- Loss: 3.1790
+- Perplexity: 24.0231
 
 ## Model description
 
@@ -69,10 +69,10 @@ The following hyperparameters were used during training:
 | 3.1212        | 4.2088 | 7500  | 0.4161   | 0.1325 | 3.2262          | 25.1831    |
 | 3.0816        | 4.4893 | 8000  | 0.4176   | 0.1307 | 3.2128          | 24.8480    |
 | 3.0917        | 4.7699 | 8500  | 0.4196   | 0.1339 | 3.1985          | 24.4954    |
-| 3.0562        | 5.0505 | 9000  | 3.2049   | 0.4185 | 24.6521         | 0.1326     |
-| 3.0683        | 5.3311 | 9500  | 3.1970   | 0.4195 | 24.4597         | 0.1307     |
-| 3.0502        | 5.6117 | 10000 | 3.1857   | 0.4209 | 24.1847         | 0.1331     |
-| 3.0469        | 5.8923 | 10500 | 3.1790   | 0.4217 | 24.0231         | 0.1309     |
+| 3.0562        | 5.0505 | 9000  | 0.4185   | 0.1326 | 3.2049          | 24.6521    |
+| 3.0683        | 5.3311 | 9500  | 0.4195   | 0.1307 | 3.1970          | 24.4597    |
+| 3.0502        | 5.6117 | 10000 | 0.4209   | 0.1331 | 3.1857          | 24.1847    |
+| 3.0469        | 5.8923 | 10500 | 0.4217   | 0.1309 | 3.1790          | 24.0231    |
 
 
 ### Framework versions

all_results.json

@@ -1,17 +1,17 @@
 {
-    "epoch": 5.0,
-    "eval_accuracy": 0.41997167684877956,
-    "eval_bleu": 0.13436705184218095,
-    "eval_loss": 3.19417405128479,
-    "eval_perplexity": 24.39002046460787,
-    "eval_runtime": 12.522,
+    "epoch": 6.0,
+    "eval_accuracy": 0.4217896359198556,
+    "eval_bleu": 0.1323262570770995,
+    "eval_loss": 3.1777124404907227,
+    "eval_perplexity": 23.991808041831927,
+    "eval_runtime": 12.2078,
     "eval_samples": 1141,
-    "eval_samples_per_second": 91.119,
-    "eval_steps_per_second": 1.437,
-    "perplexity": 24.39002046460787,
-    "total_flos": 1.0587061010143642e+18,
-    "train_loss": 0.0,
-    "train_runtime": 0.6821,
-    "train_samples_per_second": 835582.505,
-    "train_steps_per_second": 13062.161
+    "eval_samples_per_second": 93.465,
+    "eval_steps_per_second": 1.474,
+    "perplexity": 23.991808041831927,
+    "total_flos": 1.270447321217237e+18,
+    "train_loss": 0.5092627385828409,
+    "train_runtime": 3159.8581,
+    "train_samples_per_second": 216.454,
+    "train_steps_per_second": 3.384
 }

eval_results.json

@@ -1,12 +1,12 @@
 {
-    "epoch": 5.0,
-    "eval_accuracy": 0.41997167684877956,
-    "eval_bleu": 0.13436705184218095,
-    "eval_loss": 3.19417405128479,
-    "eval_perplexity": 24.39002046460787,
-    "eval_runtime": 12.522,
+    "epoch": 6.0,
+    "eval_accuracy": 0.4217896359198556,
+    "eval_bleu": 0.1323262570770995,
+    "eval_loss": 3.1777124404907227,
+    "eval_perplexity": 23.991808041831927,
+    "eval_runtime": 12.2078,
     "eval_samples": 1141,
-    "eval_samples_per_second": 91.119,
-    "eval_steps_per_second": 1.437,
-    "perplexity": 24.39002046460787
+    "eval_samples_per_second": 93.465,
+    "eval_steps_per_second": 1.474,
+    "perplexity": 23.991808041831927
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 5.0,
-    "total_flos": 1.0587061010143642e+18,
-    "train_loss": 0.0,
-    "train_runtime": 0.6821,
-    "train_samples_per_second": 835582.505,
-    "train_steps_per_second": 13062.161
+    "epoch": 6.0,
+    "total_flos": 1.270447321217237e+18,
+    "train_loss": 0.5092627385828409,
+    "train_runtime": 3159.8581,
+    "train_samples_per_second": 216.454,
+    "train_steps_per_second": 3.384
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 3.1984846591949463,
-  "best_model_checkpoint": "./output/models/rotating-head-gp-gpt2-medium-wikitext/checkpoint-8500",
-  "epoch": 5.0,
+  "best_metric": 3.1790146827697754,
+  "best_model_checkpoint": "./output/models/rotating-head-gp-gpt2-medium-wikitext/checkpoint-10500",
+  "epoch": 6.0,
   "eval_steps": 500,
-  "global_step": 8910,
+  "global_step": 10692,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -819,19 +819,182 @@
       "step": 8900
     },
     {
-      "epoch": 5.0,
-      "step": 8910,
-      "total_flos": 1.0587061010143642e+18,
-      "train_loss": 0.0,
-      "train_runtime": 0.6821,
-      "train_samples_per_second": 835582.505,
-      "train_steps_per_second": 13062.161
+      "epoch": 5.05050505050505,
+      "grad_norm": 1.2899984121322632,
+      "learning_rate": 1.7584701725213056e-05,
+      "loss": 3.0562,
+      "step": 9000
+    },
+    {
+      "epoch": 5.05050505050505,
+      "eval_accuracy": 0.4185418117735553,
+      "eval_bleu": 0.13262877395418665,
+      "eval_loss": 3.2048630714416504,
+      "eval_perplexity": 24.652124205511342,
+      "eval_runtime": 12.0913,
+      "eval_samples_per_second": 94.365,
+      "eval_steps_per_second": 1.489,
+      "step": 9000
+    },
+    {
+      "epoch": 5.10662177328844,
+      "grad_norm": 1.1415753364562988,
+      "learning_rate": 1.654541675327375e-05,
+      "loss": 3.0602,
+      "step": 9100
+    },
+    {
+      "epoch": 5.162738496071829,
+      "grad_norm": 1.3813053369522095,
+      "learning_rate": 1.5506131781334445e-05,
+      "loss": 3.0627,
+      "step": 9200
+    },
+    {
+      "epoch": 5.218855218855219,
+      "grad_norm": 1.9901399612426758,
+      "learning_rate": 1.4466846809395137e-05,
+      "loss": 3.0647,
+      "step": 9300
+    },
+    {
+      "epoch": 5.274971941638609,
+      "grad_norm": 1.109480857849121,
+      "learning_rate": 1.3427561837455832e-05,
+      "loss": 3.0496,
+      "step": 9400
+    },
+    {
+      "epoch": 5.331088664421998,
+      "grad_norm": 1.1226487159729004,
+      "learning_rate": 1.2388276865516525e-05,
+      "loss": 3.0683,
+      "step": 9500
+    },
+    {
+      "epoch": 5.331088664421998,
+      "eval_accuracy": 0.4195261826372058,
+      "eval_bleu": 0.1307104242557084,
+      "eval_loss": 3.197028875350952,
+      "eval_perplexity": 24.459749166234648,
+      "eval_runtime": 12.1714,
+      "eval_samples_per_second": 93.745,
+      "eval_steps_per_second": 1.479,
+      "step": 9500
+    },
+    {
+      "epoch": 5.3872053872053876,
+      "grad_norm": 1.8179054260253906,
+      "learning_rate": 1.134899189357722e-05,
+      "loss": 3.0641,
+      "step": 9600
+    },
+    {
+      "epoch": 5.4433221099887765,
+      "grad_norm": 1.5777114629745483,
+      "learning_rate": 1.0309706921637914e-05,
+      "loss": 3.0545,
+      "step": 9700
+    },
+    {
+      "epoch": 5.499438832772166,
+      "grad_norm": 2.161355972290039,
+      "learning_rate": 9.270421949698607e-06,
+      "loss": 3.057,
+      "step": 9800
+    },
+    {
+      "epoch": 5.555555555555555,
+      "grad_norm": 2.167677640914917,
+      "learning_rate": 8.231136977759301e-06,
+      "loss": 3.0532,
+      "step": 9900
+    },
+    {
+      "epoch": 5.611672278338945,
+      "grad_norm": 0.995482325553894,
+      "learning_rate": 7.1918520058199965e-06,
+      "loss": 3.0502,
+      "step": 10000
+    },
+    {
+      "epoch": 5.611672278338945,
+      "eval_accuracy": 0.4209234923661997,
+      "eval_bleu": 0.13307821076833265,
+      "eval_loss": 3.1857192516326904,
+      "eval_perplexity": 24.18467702013368,
+      "eval_runtime": 12.2734,
+      "eval_samples_per_second": 92.965,
+      "eval_steps_per_second": 1.467,
+      "step": 10000
+    },
+    {
+      "epoch": 5.667789001122334,
+      "grad_norm": 1.5987473726272583,
+      "learning_rate": 6.152567033880691e-06,
+      "loss": 3.0525,
+      "step": 10100
+    },
+    {
+      "epoch": 5.723905723905724,
+      "grad_norm": 1.1128613948822021,
+      "learning_rate": 5.113282061941385e-06,
+      "loss": 3.0491,
+      "step": 10200
+    },
+    {
+      "epoch": 5.780022446689113,
+      "grad_norm": 1.4610397815704346,
+      "learning_rate": 4.073997090002079e-06,
+      "loss": 3.0498,
+      "step": 10300
+    },
+    {
+      "epoch": 5.836139169472503,
+      "grad_norm": 1.4574775695800781,
+      "learning_rate": 3.0347121180627732e-06,
+      "loss": 3.0475,
+      "step": 10400
+    },
+    {
+      "epoch": 5.892255892255893,
+      "grad_norm": 1.4116820096969604,
+      "learning_rate": 1.9954271461234673e-06,
+      "loss": 3.0469,
+      "step": 10500
+    },
+    {
+      "epoch": 5.892255892255893,
+      "eval_accuracy": 0.42165513093674584,
+      "eval_bleu": 0.13088786550672932,
+      "eval_loss": 3.1790146827697754,
+      "eval_perplexity": 24.023071540521666,
+      "eval_runtime": 12.1539,
+      "eval_samples_per_second": 93.88,
+      "eval_steps_per_second": 1.481,
+      "step": 10500
+    },
+    {
+      "epoch": 5.948372615039282,
+      "grad_norm": 1.1517071723937988,
+      "learning_rate": 9.561421741841614e-07,
+      "loss": 3.0538,
+      "step": 10600
+    },
+    {
+      "epoch": 6.0,
+      "step": 10692,
+      "total_flos": 1.270447321217237e+18,
+      "train_loss": 0.5092627385828409,
+      "train_runtime": 3159.8581,
+      "train_samples_per_second": 216.454,
+      "train_steps_per_second": 3.384
     }
   ],
   "logging_steps": 100,
-  "max_steps": 8910,
+  "max_steps": 10692,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 6,
   "save_steps": 500,
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
@@ -854,7 +1017,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.0587061010143642e+18,
+  "total_flos": 1.270447321217237e+18,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04d6437c2ee925975d79e817dff86d018d08d52bce678fa75ae1d31d316d4699
+oid sha256:59200eb8b86a27533dadf7d6faf0a6fa80382d582aa8060a5ac3329ab756818f
 size 5560