Model save

README.md

@@ -142,9 +142,9 @@ The following hyperparameters were used during training:
 | 3.1393        | 9.7035  | 90000 | 0.3945   | 3.3022          |
 | 3.1482        | 9.8113  | 91000 | 0.3948   | 3.3000          |
 | 3.1491        | 9.9191  | 92000 | 0.3950   | 3.2984          |
-| 3.3065        | 10.0270 | 93000 | 3.4581   | 0.3773          |
-| 3.3805        | 10.1348 | 94000 | 3.4696   | 0.3763          |
-| 3.3884        | 10.2426 | 95000 | 3.4676   | 0.3760          |
+| 3.3064        | 10.0270 | 93000 | 3.4581   | 0.3773          |
+| 3.3807        | 10.1348 | 94000 | 3.4690   | 0.3764          |
+| 3.3886        | 10.2426 | 95000 | 3.4676   | 0.3760          |
 
 
 ### Framework versions

all_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 10.0,
+    "epoch": 10.242587601078167,
     "eval_accuracy": 0.3945371066453084,
     "eval_loss": 3.302194118499756,
-    "eval_runtime": 144.899,
+    "eval_runtime": 147.1746,
     "eval_samples": 18011,
-    "eval_samples_per_second": 124.3,
-    "eval_steps_per_second": 7.771,
+    "eval_samples_per_second": 122.378,
+    "eval_steps_per_second": 7.651,
     "perplexity": 27.17219257328467,
-    "total_flos": 7.75449427968e+17,
-    "train_loss": 3.4695698613991954,
-    "train_runtime": 61247.9496,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07955799303556743,
+    "train_runtime": 1634.6435,
     "train_samples": 296775,
-    "train_samples_per_second": 48.455,
-    "train_steps_per_second": 1.514
+    "train_samples_per_second": 9077.667,
+    "train_steps_per_second": 283.701
 }

eval_results.json

@@ -1,10 +1,10 @@
 {
-    "epoch": 10.0,
+    "epoch": 10.242587601078167,
     "eval_accuracy": 0.3945371066453084,
     "eval_loss": 3.302194118499756,
-    "eval_runtime": 144.899,
+    "eval_runtime": 147.1746,
     "eval_samples": 18011,
-    "eval_samples_per_second": 124.3,
-    "eval_steps_per_second": 7.771,
+    "eval_samples_per_second": 122.378,
+    "eval_steps_per_second": 7.651,
     "perplexity": 27.17219257328467
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 10.0,
-    "total_flos": 7.75449427968e+17,
-    "train_loss": 3.4695698613991954,
-    "train_runtime": 61247.9496,
+    "epoch": 10.242587601078167,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07955799303556743,
+    "train_runtime": 1634.6435,
     "train_samples": 296775,
-    "train_samples_per_second": 48.455,
-    "train_steps_per_second": 1.514
+    "train_samples_per_second": 9077.667,
+    "train_steps_per_second": 283.701
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 3.302194118499756,
   "best_model_checkpoint": "/scratch/cl5625/exceptions/models/100M__495/checkpoint-90000",
-  "epoch": 10.0,
+  "epoch": 10.242587601078167,
   "eval_steps": 1000,
-  "global_step": 92750,
+  "global_step": 95000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -13822,19 +13822,361 @@
       "step": 92750
     },
     {
-      "epoch": 10.0,
-      "step": 92750,
-      "total_flos": 7.75449427968e+17,
-      "train_loss": 3.4695698613991954,
-      "train_runtime": 61247.9496,
-      "train_samples_per_second": 48.455,
-      "train_steps_per_second": 1.514
+      "epoch": 10.005390835579515,
+      "grad_norm": 1.025435209274292,
+      "learning_rate": 0.0004800879974118408,
+      "loss": 3.1767,
+      "step": 92800
+    },
+    {
+      "epoch": 10.01078167115903,
+      "grad_norm": 1.0077670812606812,
+      "learning_rate": 0.00048002329343254605,
+      "loss": 3.2428,
+      "step": 92850
+    },
+    {
+      "epoch": 10.016172506738544,
+      "grad_norm": 1.0313729047775269,
+      "learning_rate": 0.0004799585894532513,
+      "loss": 3.277,
+      "step": 92900
+    },
+    {
+      "epoch": 10.021563342318059,
+      "grad_norm": 0.9879314303398132,
+      "learning_rate": 0.0004798938854739566,
+      "loss": 3.294,
+      "step": 92950
+    },
+    {
+      "epoch": 10.026954177897574,
+      "grad_norm": 1.1394336223602295,
+      "learning_rate": 0.0004798291814946619,
+      "loss": 3.3065,
+      "step": 93000
+    },
+    {
+      "epoch": 10.026954177897574,
+      "eval_accuracy": 0.3772760742755487,
+      "eval_loss": 3.4581103324890137,
+      "eval_runtime": 148.4364,
+      "eval_samples_per_second": 121.338,
+      "eval_steps_per_second": 7.586,
+      "step": 93000
+    },
+    {
+      "epoch": 10.032345013477089,
+      "grad_norm": 1.0531760454177856,
+      "learning_rate": 0.00047976447751536713,
+      "loss": 3.3043,
+      "step": 93050
+    },
+    {
+      "epoch": 10.037735849056604,
+      "grad_norm": 1.0207242965698242,
+      "learning_rate": 0.0004796997735360724,
+      "loss": 3.3286,
+      "step": 93100
+    },
+    {
+      "epoch": 10.04312668463612,
+      "grad_norm": 0.9906688332557678,
+      "learning_rate": 0.0004796350695567777,
+      "loss": 3.329,
+      "step": 93150
+    },
+    {
+      "epoch": 10.048517520215633,
+      "grad_norm": 0.9772278070449829,
+      "learning_rate": 0.00047957036557748296,
+      "loss": 3.34,
+      "step": 93200
+    },
+    {
+      "epoch": 10.053908355795148,
+      "grad_norm": 1.0326145887374878,
+      "learning_rate": 0.00047950566159818826,
+      "loss": 3.3329,
+      "step": 93250
+    },
+    {
+      "epoch": 10.059299191374663,
+      "grad_norm": 1.0279661417007446,
+      "learning_rate": 0.0004794409576188935,
+      "loss": 3.3593,
+      "step": 93300
+    },
+    {
+      "epoch": 10.064690026954178,
+      "grad_norm": 1.0010513067245483,
+      "learning_rate": 0.0004793762536395988,
+      "loss": 3.3611,
+      "step": 93350
+    },
+    {
+      "epoch": 10.070080862533693,
+      "grad_norm": 0.8727930188179016,
+      "learning_rate": 0.0004793115496603041,
+      "loss": 3.3464,
+      "step": 93400
+    },
+    {
+      "epoch": 10.075471698113208,
+      "grad_norm": 0.928733766078949,
+      "learning_rate": 0.00047924684568100934,
+      "loss": 3.3566,
+      "step": 93450
+    },
+    {
+      "epoch": 10.080862533692722,
+      "grad_norm": 0.9133866429328918,
+      "learning_rate": 0.0004791821417017146,
+      "loss": 3.3484,
+      "step": 93500
+    },
+    {
+      "epoch": 10.086253369272237,
+      "grad_norm": 0.9412495493888855,
+      "learning_rate": 0.00047911743772241993,
+      "loss": 3.3635,
+      "step": 93550
+    },
+    {
+      "epoch": 10.091644204851752,
+      "grad_norm": 0.9078878164291382,
+      "learning_rate": 0.00047905273374312517,
+      "loss": 3.3718,
+      "step": 93600
+    },
+    {
+      "epoch": 10.097035040431267,
+      "grad_norm": 0.9754884243011475,
+      "learning_rate": 0.0004789880297638304,
+      "loss": 3.3655,
+      "step": 93650
+    },
+    {
+      "epoch": 10.102425876010782,
+      "grad_norm": 0.9449187517166138,
+      "learning_rate": 0.00047892332578453576,
+      "loss": 3.3622,
+      "step": 93700
+    },
+    {
+      "epoch": 10.107816711590296,
+      "grad_norm": 0.970542848110199,
+      "learning_rate": 0.000478858621805241,
+      "loss": 3.3786,
+      "step": 93750
+    },
+    {
+      "epoch": 10.11320754716981,
+      "grad_norm": 0.990994393825531,
+      "learning_rate": 0.00047879391782594625,
+      "loss": 3.3696,
+      "step": 93800
+    },
+    {
+      "epoch": 10.118598382749326,
+      "grad_norm": 0.8434808850288391,
+      "learning_rate": 0.0004787292138466515,
+      "loss": 3.3601,
+      "step": 93850
+    },
+    {
+      "epoch": 10.123989218328841,
+      "grad_norm": 0.955342710018158,
+      "learning_rate": 0.00047866450986735684,
+      "loss": 3.3868,
+      "step": 93900
+    },
+    {
+      "epoch": 10.129380053908356,
+      "grad_norm": 0.8750738501548767,
+      "learning_rate": 0.0004785998058880621,
+      "loss": 3.3788,
+      "step": 93950
+    },
+    {
+      "epoch": 10.134770889487871,
+      "grad_norm": 0.9050654768943787,
+      "learning_rate": 0.0004785351019087673,
+      "loss": 3.3805,
+      "step": 94000
+    },
+    {
+      "epoch": 10.134770889487871,
+      "eval_accuracy": 0.3762606043860346,
+      "eval_loss": 3.4695773124694824,
+      "eval_runtime": 148.5415,
+      "eval_samples_per_second": 121.252,
+      "eval_steps_per_second": 7.58,
+      "step": 94000
+    },
+    {
+      "epoch": 10.140161725067385,
+      "grad_norm": 0.9092472791671753,
+      "learning_rate": 0.00047847039792947257,
+      "loss": 3.3753,
+      "step": 94050
+    },
+    {
+      "epoch": 10.1455525606469,
+      "grad_norm": 0.913253128528595,
+      "learning_rate": 0.0004784056939501779,
+      "loss": 3.379,
+      "step": 94100
+    },
+    {
+      "epoch": 10.150943396226415,
+      "grad_norm": 0.9133390784263611,
+      "learning_rate": 0.00047834098997088316,
+      "loss": 3.3794,
+      "step": 94150
+    },
+    {
+      "epoch": 10.15633423180593,
+      "grad_norm": 0.8519137501716614,
+      "learning_rate": 0.0004782762859915884,
+      "loss": 3.3925,
+      "step": 94200
+    },
+    {
+      "epoch": 10.161725067385445,
+      "grad_norm": 0.9071498513221741,
+      "learning_rate": 0.0004782115820122937,
+      "loss": 3.3714,
+      "step": 94250
+    },
+    {
+      "epoch": 10.167115902964959,
+      "grad_norm": 0.868298351764679,
+      "learning_rate": 0.000478146878032999,
+      "loss": 3.3912,
+      "step": 94300
+    },
+    {
+      "epoch": 10.172506738544474,
+      "grad_norm": 0.8858316540718079,
+      "learning_rate": 0.00047808217405370424,
+      "loss": 3.363,
+      "step": 94350
+    },
+    {
+      "epoch": 10.177897574123989,
+      "grad_norm": 0.820442259311676,
+      "learning_rate": 0.00047801747007440953,
+      "loss": 3.3963,
+      "step": 94400
+    },
+    {
+      "epoch": 10.183288409703504,
+      "grad_norm": 0.9085699319839478,
+      "learning_rate": 0.00047795276609511483,
+      "loss": 3.3966,
+      "step": 94450
+    },
+    {
+      "epoch": 10.18867924528302,
+      "grad_norm": 0.8699147701263428,
+      "learning_rate": 0.00047788806211582007,
+      "loss": 3.3895,
+      "step": 94500
+    },
+    {
+      "epoch": 10.194070080862534,
+      "grad_norm": 0.8835978507995605,
+      "learning_rate": 0.00047782335813652537,
+      "loss": 3.3917,
+      "step": 94550
+    },
+    {
+      "epoch": 10.199460916442048,
+      "grad_norm": 0.8378937244415283,
+      "learning_rate": 0.0004777586541572306,
+      "loss": 3.3785,
+      "step": 94600
+    },
+    {
+      "epoch": 10.204851752021563,
+      "grad_norm": 0.8834420442581177,
+      "learning_rate": 0.0004776939501779359,
+      "loss": 3.3843,
+      "step": 94650
+    },
+    {
+      "epoch": 10.210242587601078,
+      "grad_norm": 0.9476656913757324,
+      "learning_rate": 0.0004776292461986412,
+      "loss": 3.394,
+      "step": 94700
+    },
+    {
+      "epoch": 10.215633423180593,
+      "grad_norm": 0.8962563872337341,
+      "learning_rate": 0.00047756454221934645,
+      "loss": 3.3882,
+      "step": 94750
+    },
+    {
+      "epoch": 10.221024258760108,
+      "grad_norm": 0.8866973519325256,
+      "learning_rate": 0.0004774998382400517,
+      "loss": 3.3908,
+      "step": 94800
+    },
+    {
+      "epoch": 10.226415094339623,
+      "grad_norm": 0.8504216074943542,
+      "learning_rate": 0.00047743513426075704,
+      "loss": 3.3871,
+      "step": 94850
+    },
+    {
+      "epoch": 10.231805929919137,
+      "grad_norm": 0.8385003805160522,
+      "learning_rate": 0.0004773704302814623,
+      "loss": 3.3962,
+      "step": 94900
+    },
+    {
+      "epoch": 10.237196765498652,
+      "grad_norm": 0.8448400497436523,
+      "learning_rate": 0.0004773057263021675,
+      "loss": 3.4058,
+      "step": 94950
+    },
+    {
+      "epoch": 10.242587601078167,
+      "grad_norm": 0.8619194626808167,
+      "learning_rate": 0.0004772410223228728,
+      "loss": 3.3884,
+      "step": 95000
+    },
+    {
+      "epoch": 10.242587601078167,
+      "eval_accuracy": 0.37598745102606895,
+      "eval_loss": 3.4676356315612793,
+      "eval_runtime": 146.923,
+      "eval_samples_per_second": 122.588,
+      "eval_steps_per_second": 7.664,
+      "step": 95000
+    },
+    {
+      "epoch": 10.242587601078167,
+      "step": 95000,
+      "total_flos": 7.94262454272e+17,
+      "train_loss": 0.07955799303556743,
+      "train_runtime": 1634.6435,
+      "train_samples_per_second": 9077.667,
+      "train_steps_per_second": 283.701
     }
   ],
   "logging_steps": 50,
-  "max_steps": 92750,
+  "max_steps": 463750,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 10,
+  "num_train_epochs": 50,
   "save_steps": 10000,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -13848,7 +14190,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.75449427968e+17,
+  "total_flos": 7.94262454272e+17,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d24f4ca1b52c9db2ed02b327a68a7c52c020ab559dd4b89a1344f00db4d14cb
+oid sha256:e7a4f9fe5e6c6a0f0bfabe88acf34319237a3bdae6f2c87408262f5977a18ecd
 size 5304