Model save

README.md

@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 3.4624
+- Loss: 3.4625
 - Accuracy: 0.3763
 
 ## Model description
@@ -143,8 +143,8 @@ The following hyperparameters were used during training:
 | 3.1314        | 9.8113  | 91000 | 0.3943   | 3.3021          |
 | 3.1061        | 9.9191  | 92000 | 0.3946   | 3.3004          |
 | 3.2987        | 10.0270 | 93000 | 3.4598   | 0.3772          |
-| 3.3802        | 10.1348 | 94000 | 3.4702   | 0.3756          |
-| 3.3824        | 10.2426 | 95000 | 3.4624   | 0.3763          |
+| 3.3802        | 10.1348 | 94000 | 3.4705   | 0.3756          |
+| 3.3831        | 10.2426 | 95000 | 3.4625   | 0.3763          |
 
 
 ### Framework versions

all_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 10.0,
-    "eval_accuracy": 0.39428448868113974,
-    "eval_loss": 3.3030037879943848,
-    "eval_runtime": 181.4646,
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3942879655735498,
+    "eval_loss": 3.303001642227173,
+    "eval_runtime": 145.5138,
     "eval_samples": 18011,
-    "eval_samples_per_second": 99.254,
-    "eval_steps_per_second": 6.205,
-    "perplexity": 27.194201977682727,
-    "total_flos": 7.75449427968e+17,
-    "train_loss": 3.4548003072815763,
-    "train_runtime": 79154.1696,
+    "eval_samples_per_second": 123.775,
+    "eval_steps_per_second": 7.738,
+    "perplexity": 27.19414362531837,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07928502052708676,
+    "train_runtime": 1615.2002,
     "train_samples": 296775,
-    "train_samples_per_second": 37.493,
-    "train_steps_per_second": 1.172
+    "train_samples_per_second": 9186.942,
+    "train_steps_per_second": 287.116
 }

eval_results.json

@@ -1,10 +1,10 @@
 {
-    "epoch": 10.0,
-    "eval_accuracy": 0.39428448868113974,
-    "eval_loss": 3.3030037879943848,
-    "eval_runtime": 181.4646,
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.3942879655735498,
+    "eval_loss": 3.303001642227173,
+    "eval_runtime": 145.5138,
     "eval_samples": 18011,
-    "eval_samples_per_second": 99.254,
-    "eval_steps_per_second": 6.205,
-    "perplexity": 27.194201977682727
+    "eval_samples_per_second": 123.775,
+    "eval_steps_per_second": 7.738,
+    "perplexity": 27.19414362531837
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 10.0,
-    "total_flos": 7.75449427968e+17,
-    "train_loss": 3.4548003072815763,
-    "train_runtime": 79154.1696,
+    "epoch": 10.242587601078167,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07928502052708676,
+    "train_runtime": 1615.2002,
     "train_samples": 296775,
-    "train_samples_per_second": 37.493,
-    "train_steps_per_second": 1.172
+    "train_samples_per_second": 9186.942,
+    "train_steps_per_second": 287.116
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 3.3030037879943848,
   "best_model_checkpoint": "/scratch/cl5625/exceptions/models/100M__634/checkpoint-90000",
-  "epoch": 10.0,
+  "epoch": 10.242587601078167,
   "eval_steps": 1000,
-  "global_step": 92750,
+  "global_step": 95000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -13822,19 +13822,361 @@
       "step": 92750
     },
     {
-      "epoch": 10.0,
-      "step": 92750,
-      "total_flos": 7.75449427968e+17,
-      "train_loss": 3.4548003072815763,
-      "train_runtime": 79154.1696,
-      "train_samples_per_second": 37.493,
-      "train_steps_per_second": 1.172
+      "epoch": 10.005390835579515,
+      "grad_norm": 1.0576950311660767,
+      "learning_rate": 0.000480085409252669,
+      "loss": 3.1663,
+      "step": 92800
+    },
+    {
+      "epoch": 10.01078167115903,
+      "grad_norm": 1.019791603088379,
+      "learning_rate": 0.00048002070527337426,
+      "loss": 3.2216,
+      "step": 92850
+    },
+    {
+      "epoch": 10.016172506738544,
+      "grad_norm": 0.9198017120361328,
+      "learning_rate": 0.00047995600129407956,
+      "loss": 3.2466,
+      "step": 92900
+    },
+    {
+      "epoch": 10.021563342318059,
+      "grad_norm": 1.101521372795105,
+      "learning_rate": 0.0004798912973147848,
+      "loss": 3.2908,
+      "step": 92950
+    },
+    {
+      "epoch": 10.026954177897574,
+      "grad_norm": 1.0034935474395752,
+      "learning_rate": 0.0004798265933354901,
+      "loss": 3.2987,
+      "step": 93000
+    },
+    {
+      "epoch": 10.026954177897574,
+      "eval_accuracy": 0.37723413426085234,
+      "eval_loss": 3.459772825241089,
+      "eval_runtime": 146.5373,
+      "eval_samples_per_second": 122.911,
+      "eval_steps_per_second": 7.684,
+      "step": 93000
+    },
+    {
+      "epoch": 10.032345013477089,
+      "grad_norm": 0.9505324363708496,
+      "learning_rate": 0.0004797618893561954,
+      "loss": 3.2895,
+      "step": 93050
+    },
+    {
+      "epoch": 10.037735849056604,
+      "grad_norm": 1.004460334777832,
+      "learning_rate": 0.00047969718537690064,
+      "loss": 3.3141,
+      "step": 93100
+    },
+    {
+      "epoch": 10.04312668463612,
+      "grad_norm": 0.9728407263755798,
+      "learning_rate": 0.0004796324813976059,
+      "loss": 3.3228,
+      "step": 93150
+    },
+    {
+      "epoch": 10.048517520215633,
+      "grad_norm": 1.1120251417160034,
+      "learning_rate": 0.00047956777741831123,
+      "loss": 3.3313,
+      "step": 93200
+    },
+    {
+      "epoch": 10.053908355795148,
+      "grad_norm": 0.960462212562561,
+      "learning_rate": 0.00047950307343901647,
+      "loss": 3.3163,
+      "step": 93250
+    },
+    {
+      "epoch": 10.059299191374663,
+      "grad_norm": 0.9895879030227661,
+      "learning_rate": 0.0004794383694597217,
+      "loss": 3.3216,
+      "step": 93300
+    },
+    {
+      "epoch": 10.064690026954178,
+      "grad_norm": 0.9253028035163879,
+      "learning_rate": 0.00047937366548042696,
+      "loss": 3.3511,
+      "step": 93350
+    },
+    {
+      "epoch": 10.070080862533693,
+      "grad_norm": 1.0076348781585693,
+      "learning_rate": 0.0004793089615011323,
+      "loss": 3.3373,
+      "step": 93400
+    },
+    {
+      "epoch": 10.075471698113208,
+      "grad_norm": 1.0603855848312378,
+      "learning_rate": 0.00047924425752183755,
+      "loss": 3.3447,
+      "step": 93450
+    },
+    {
+      "epoch": 10.080862533692722,
+      "grad_norm": 0.9239181280136108,
+      "learning_rate": 0.0004791795535425428,
+      "loss": 3.334,
+      "step": 93500
+    },
+    {
+      "epoch": 10.086253369272237,
+      "grad_norm": 0.954738974571228,
+      "learning_rate": 0.00047911484956324814,
+      "loss": 3.3477,
+      "step": 93550
+    },
+    {
+      "epoch": 10.091644204851752,
+      "grad_norm": 0.9450059533119202,
+      "learning_rate": 0.0004790501455839534,
+      "loss": 3.3501,
+      "step": 93600
+    },
+    {
+      "epoch": 10.097035040431267,
+      "grad_norm": 0.958166241645813,
+      "learning_rate": 0.0004789854416046586,
+      "loss": 3.3689,
+      "step": 93650
+    },
+    {
+      "epoch": 10.102425876010782,
+      "grad_norm": 0.9498693943023682,
+      "learning_rate": 0.00047892073762536387,
+      "loss": 3.3668,
+      "step": 93700
+    },
+    {
+      "epoch": 10.107816711590296,
+      "grad_norm": 0.9249439835548401,
+      "learning_rate": 0.0004788560336460692,
+      "loss": 3.3628,
+      "step": 93750
+    },
+    {
+      "epoch": 10.11320754716981,
+      "grad_norm": 0.9309000372886658,
+      "learning_rate": 0.00047879132966677446,
+      "loss": 3.3635,
+      "step": 93800
+    },
+    {
+      "epoch": 10.118598382749326,
+      "grad_norm": 0.9265908002853394,
+      "learning_rate": 0.0004787266256874797,
+      "loss": 3.3733,
+      "step": 93850
+    },
+    {
+      "epoch": 10.123989218328841,
+      "grad_norm": 0.9763278961181641,
+      "learning_rate": 0.000478661921708185,
+      "loss": 3.348,
+      "step": 93900
+    },
+    {
+      "epoch": 10.129380053908356,
+      "grad_norm": 0.8944584727287292,
+      "learning_rate": 0.0004785972177288903,
+      "loss": 3.3714,
+      "step": 93950
+    },
+    {
+      "epoch": 10.134770889487871,
+      "grad_norm": 1.097298264503479,
+      "learning_rate": 0.00047853251374959554,
+      "loss": 3.3802,
+      "step": 94000
+    },
+    {
+      "epoch": 10.134770889487871,
+      "eval_accuracy": 0.3755615317058362,
+      "eval_loss": 3.470184803009033,
+      "eval_runtime": 148.0392,
+      "eval_samples_per_second": 121.664,
+      "eval_steps_per_second": 7.606,
+      "step": 94000
+    },
+    {
+      "epoch": 10.140161725067385,
+      "grad_norm": 0.893616259098053,
+      "learning_rate": 0.00047846780977030083,
+      "loss": 3.3583,
+      "step": 94050
+    },
+    {
+      "epoch": 10.1455525606469,
+      "grad_norm": 0.8614599108695984,
+      "learning_rate": 0.00047840310579100613,
+      "loss": 3.3667,
+      "step": 94100
+    },
+    {
+      "epoch": 10.150943396226415,
+      "grad_norm": 0.9499636292457581,
+      "learning_rate": 0.00047833840181171137,
+      "loss": 3.3603,
+      "step": 94150
+    },
+    {
+      "epoch": 10.15633423180593,
+      "grad_norm": 0.9031397104263306,
+      "learning_rate": 0.00047827369783241667,
+      "loss": 3.3549,
+      "step": 94200
+    },
+    {
+      "epoch": 10.161725067385445,
+      "grad_norm": 0.8364366888999939,
+      "learning_rate": 0.0004782089938531219,
+      "loss": 3.3738,
+      "step": 94250
+    },
+    {
+      "epoch": 10.167115902964959,
+      "grad_norm": 0.8245320320129395,
+      "learning_rate": 0.0004781442898738272,
+      "loss": 3.3764,
+      "step": 94300
+    },
+    {
+      "epoch": 10.172506738544474,
+      "grad_norm": 0.8197740316390991,
+      "learning_rate": 0.0004780795858945325,
+      "loss": 3.3841,
+      "step": 94350
+    },
+    {
+      "epoch": 10.177897574123989,
+      "grad_norm": 0.8775398135185242,
+      "learning_rate": 0.00047801488191523775,
+      "loss": 3.3669,
+      "step": 94400
+    },
+    {
+      "epoch": 10.183288409703504,
+      "grad_norm": 0.9364191889762878,
+      "learning_rate": 0.000477950177935943,
+      "loss": 3.3592,
+      "step": 94450
+    },
+    {
+      "epoch": 10.18867924528302,
+      "grad_norm": 0.8914583921432495,
+      "learning_rate": 0.00047788547395664834,
+      "loss": 3.3755,
+      "step": 94500
+    },
+    {
+      "epoch": 10.194070080862534,
+      "grad_norm": 0.7897307872772217,
+      "learning_rate": 0.0004778207699773536,
+      "loss": 3.3799,
+      "step": 94550
+    },
+    {
+      "epoch": 10.199460916442048,
+      "grad_norm": 0.83349609375,
+      "learning_rate": 0.0004777560659980588,
+      "loss": 3.3733,
+      "step": 94600
+    },
+    {
+      "epoch": 10.204851752021563,
+      "grad_norm": 0.9644209146499634,
+      "learning_rate": 0.00047769136201876406,
+      "loss": 3.3894,
+      "step": 94650
+    },
+    {
+      "epoch": 10.210242587601078,
+      "grad_norm": 0.7957009077072144,
+      "learning_rate": 0.0004776266580394694,
+      "loss": 3.3855,
+      "step": 94700
+    },
+    {
+      "epoch": 10.215633423180593,
+      "grad_norm": 0.8263417482376099,
+      "learning_rate": 0.00047756195406017466,
+      "loss": 3.3857,
+      "step": 94750
+    },
+    {
+      "epoch": 10.221024258760108,
+      "grad_norm": 0.8626648187637329,
+      "learning_rate": 0.0004774972500808799,
+      "loss": 3.3935,
+      "step": 94800
+    },
+    {
+      "epoch": 10.226415094339623,
+      "grad_norm": 0.8164050579071045,
+      "learning_rate": 0.00047743254610158525,
+      "loss": 3.3991,
+      "step": 94850
+    },
+    {
+      "epoch": 10.231805929919137,
+      "grad_norm": 0.9200409054756165,
+      "learning_rate": 0.0004773678421222905,
+      "loss": 3.3668,
+      "step": 94900
+    },
+    {
+      "epoch": 10.237196765498652,
+      "grad_norm": 0.7908764481544495,
+      "learning_rate": 0.00047730313814299573,
+      "loss": 3.3905,
+      "step": 94950
+    },
+    {
+      "epoch": 10.242587601078167,
+      "grad_norm": 0.7977242469787598,
+      "learning_rate": 0.000477238434163701,
+      "loss": 3.3824,
+      "step": 95000
+    },
+    {
+      "epoch": 10.242587601078167,
+      "eval_accuracy": 0.376265059154435,
+      "eval_loss": 3.4624273777008057,
+      "eval_runtime": 146.0721,
+      "eval_samples_per_second": 123.302,
+      "eval_steps_per_second": 7.709,
+      "step": 95000
+    },
+    {
+      "epoch": 10.242587601078167,
+      "step": 95000,
+      "total_flos": 7.94262454272e+17,
+      "train_loss": 0.07928502052708676,
+      "train_runtime": 1615.2002,
+      "train_samples_per_second": 9186.942,
+      "train_steps_per_second": 287.116
     }
   ],
   "logging_steps": 50,
-  "max_steps": 92750,
+  "max_steps": 463750,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 10,
+  "num_train_epochs": 50,
   "save_steps": 10000,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -13848,7 +14190,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.75449427968e+17,
+  "total_flos": 7.94262454272e+17,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dbba059ae4e965260f4be9ebe73f66dc0ec9b205428e29ea65d2a6a6ded9fe2
+oid sha256:ed851c4d340fa66276b0dfb5ef19368092eb830abe5fc6c31e1a867a4b565c42
 size 5304