Training in progress, epoch 0

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 24.137931034482758,
-    "total_flos": 8.67232581632557e+17,
-    "train_loss": 0.1937425845861435,
-    "train_runtime": 510.3099,
-    "train_samples_per_second": 22.29,
-    "train_steps_per_second": 0.343
+    "epoch": 19.310344827586206,
+    "total_flos": 6.939281828084982e+17,
+    "train_loss": 0.17856683683182512,
+    "train_runtime": 404.7192,
+    "train_samples_per_second": 22.485,
+    "train_steps_per_second": 0.346
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:685afda8187426a539c171c5d3c2da746ca2bcf16148aaf21ad1fae8bf31f73c
+oid sha256:1224cb50773e022de5e3d43ba646f154c63e362ac5f079be8c5990db2885b490
 size 349854120

runs/Jul24_14-20-26_gurame007/events.out.tfevents.1721805627.gurame007.11332.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e4244523c80ff4dd0472e8c1306da0e2782832458f0eff409c7599732dc872e
+size 5118

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 24.137931034482758,
-    "total_flos": 8.67232581632557e+17,
-    "train_loss": 0.1937425845861435,
-    "train_runtime": 510.3099,
-    "train_samples_per_second": 22.29,
-    "train_steps_per_second": 0.343
+    "epoch": 19.310344827586206,
+    "total_flos": 6.939281828084982e+17,
+    "train_loss": 0.17856683683182512,
+    "train_runtime": 404.7192,
+    "train_samples_per_second": 22.485,
+    "train_steps_per_second": 0.346
 }

trainer_state.json

@@ -1,329 +1,270 @@
 {
-  "best_metric": 0.9120879120879121,
-  "best_model_checkpoint": "document-crop/checkpoint-166",
-  "epoch": 24.137931034482758,
+  "best_metric": 0.967032967032967,
+  "best_model_checkpoint": "document-crop/checkpoint-72",
+  "epoch": 19.310344827586206,
   "eval_steps": 500,
-  "global_step": 175,
+  "global_step": 140,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.9655172413793104,
-      "eval_accuracy": 0.5494505494505495,
-      "eval_loss": 0.7309015393257141,
-      "eval_runtime": 2.5551,
-      "eval_samples_per_second": 35.615,
-      "eval_steps_per_second": 2.348,
+      "eval_accuracy": 0.9120879120879121,
+      "eval_loss": 0.3032292127609253,
+      "eval_runtime": 2.5391,
+      "eval_samples_per_second": 35.839,
+      "eval_steps_per_second": 2.363,
       "step": 7
     },
     {
       "epoch": 1.9310344827586206,
-      "eval_accuracy": 0.5934065934065934,
-      "eval_loss": 0.6288828253746033,
-      "eval_runtime": 2.6587,
-      "eval_samples_per_second": 34.228,
-      "eval_steps_per_second": 2.257,
+      "eval_accuracy": 0.8241758241758241,
+      "eval_loss": 0.3624477684497833,
+      "eval_runtime": 2.5753,
+      "eval_samples_per_second": 35.335,
+      "eval_steps_per_second": 2.33,
       "step": 14
     },
     {
       "epoch": 2.0689655172413794,
-      "grad_norm": 25.187419891357422,
-      "learning_rate": 4.166666666666667e-05,
-      "loss": 0.6408,
+      "grad_norm": 17.86758804321289,
+      "learning_rate": 4.960317460317461e-05,
+      "loss": 0.48,
       "step": 15
     },
     {
       "epoch": 2.896551724137931,
-      "eval_accuracy": 0.6923076923076923,
-      "eval_loss": 0.5929833054542542,
-      "eval_runtime": 2.6282,
-      "eval_samples_per_second": 34.625,
-      "eval_steps_per_second": 2.283,
+      "eval_accuracy": 0.8351648351648352,
+      "eval_loss": 0.3741466701030731,
+      "eval_runtime": 2.5668,
+      "eval_samples_per_second": 35.453,
+      "eval_steps_per_second": 2.338,
       "step": 21
     },
     {
       "epoch": 4.0,
-      "eval_accuracy": 0.6153846153846154,
-      "eval_loss": 0.896169126033783,
-      "eval_runtime": 2.573,
-      "eval_samples_per_second": 35.368,
-      "eval_steps_per_second": 2.332,
+      "eval_accuracy": 0.8901098901098901,
+      "eval_loss": 0.23425163328647614,
+      "eval_runtime": 2.5564,
+      "eval_samples_per_second": 35.596,
+      "eval_steps_per_second": 2.347,
       "step": 29
     },
     {
       "epoch": 4.137931034482759,
-      "grad_norm": 14.1068754196167,
-      "learning_rate": 4.617834394904459e-05,
-      "loss": 0.4193,
+      "grad_norm": 19.562450408935547,
+      "learning_rate": 4.3650793650793655e-05,
+      "loss": 0.3749,
       "step": 30
     },
     {
       "epoch": 4.9655172413793105,
-      "eval_accuracy": 0.8021978021978022,
-      "eval_loss": 0.4717129170894623,
-      "eval_runtime": 2.6421,
-      "eval_samples_per_second": 34.442,
-      "eval_steps_per_second": 2.271,
+      "eval_accuracy": 0.9120879120879121,
+      "eval_loss": 0.20266202092170715,
+      "eval_runtime": 2.6328,
+      "eval_samples_per_second": 34.564,
+      "eval_steps_per_second": 2.279,
       "step": 36
     },
     {
       "epoch": 5.931034482758621,
-      "eval_accuracy": 0.8241758241758241,
-      "eval_loss": 0.4143160581588745,
-      "eval_runtime": 2.7032,
-      "eval_samples_per_second": 33.663,
-      "eval_steps_per_second": 2.22,
+      "eval_accuracy": 0.945054945054945,
+      "eval_loss": 0.19039569795131683,
+      "eval_runtime": 2.6292,
+      "eval_samples_per_second": 34.611,
+      "eval_steps_per_second": 2.282,
       "step": 43
     },
     {
       "epoch": 6.206896551724138,
-      "grad_norm": 17.542375564575195,
-      "learning_rate": 4.1401273885350325e-05,
-      "loss": 0.3622,
+      "grad_norm": 13.474059104919434,
+      "learning_rate": 3.76984126984127e-05,
+      "loss": 0.2323,
       "step": 45
     },
     {
       "epoch": 6.896551724137931,
-      "eval_accuracy": 0.8241758241758241,
-      "eval_loss": 0.4415420591831207,
-      "eval_runtime": 2.674,
-      "eval_samples_per_second": 34.031,
-      "eval_steps_per_second": 2.244,
+      "eval_accuracy": 0.9120879120879121,
+      "eval_loss": 0.19586102664470673,
+      "eval_runtime": 2.6461,
+      "eval_samples_per_second": 34.39,
+      "eval_steps_per_second": 2.267,
       "step": 50
     },
     {
       "epoch": 8.0,
-      "eval_accuracy": 0.8461538461538461,
-      "eval_loss": 0.3617265224456787,
-      "eval_runtime": 2.614,
-      "eval_samples_per_second": 34.813,
-      "eval_steps_per_second": 2.295,
+      "eval_accuracy": 0.8901098901098901,
+      "eval_loss": 0.2905385494232178,
+      "eval_runtime": 2.6111,
+      "eval_samples_per_second": 34.852,
+      "eval_steps_per_second": 2.298,
       "step": 58
     },
     {
       "epoch": 8.275862068965518,
-      "grad_norm": 15.421939849853516,
-      "learning_rate": 3.662420382165605e-05,
-      "loss": 0.2049,
+      "grad_norm": 24.003192901611328,
+      "learning_rate": 3.1746031746031745e-05,
+      "loss": 0.1447,
       "step": 60
     },
     {
       "epoch": 8.96551724137931,
-      "eval_accuracy": 0.8351648351648352,
-      "eval_loss": 0.4249739944934845,
-      "eval_runtime": 2.7136,
-      "eval_samples_per_second": 33.534,
-      "eval_steps_per_second": 2.211,
+      "eval_accuracy": 0.9010989010989011,
+      "eval_loss": 0.3998720347881317,
+      "eval_runtime": 2.6866,
+      "eval_samples_per_second": 33.872,
+      "eval_steps_per_second": 2.233,
       "step": 65
     },
     {
       "epoch": 9.931034482758621,
-      "eval_accuracy": 0.8461538461538461,
-      "eval_loss": 0.40621432662010193,
-      "eval_runtime": 2.6749,
-      "eval_samples_per_second": 34.02,
-      "eval_steps_per_second": 2.243,
+      "eval_accuracy": 0.967032967032967,
+      "eval_loss": 0.10719592869281769,
+      "eval_runtime": 2.6592,
+      "eval_samples_per_second": 34.22,
+      "eval_steps_per_second": 2.256,
       "step": 72
     },
     {
       "epoch": 10.344827586206897,
-      "grad_norm": 30.14670181274414,
-      "learning_rate": 3.184713375796178e-05,
-      "loss": 0.1712,
+      "grad_norm": 12.631780624389648,
+      "learning_rate": 2.5793650793650796e-05,
+      "loss": 0.125,
       "step": 75
     },
     {
       "epoch": 10.89655172413793,
-      "eval_accuracy": 0.8241758241758241,
-      "eval_loss": 0.4119836986064911,
-      "eval_runtime": 2.729,
-      "eval_samples_per_second": 33.345,
-      "eval_steps_per_second": 2.199,
+      "eval_accuracy": 0.9010989010989011,
+      "eval_loss": 0.2653524875640869,
+      "eval_runtime": 2.6798,
+      "eval_samples_per_second": 33.958,
+      "eval_steps_per_second": 2.239,
       "step": 79
     },
     {
       "epoch": 12.0,
-      "eval_accuracy": 0.8131868131868132,
-      "eval_loss": 0.5776697397232056,
-      "eval_runtime": 2.6961,
-      "eval_samples_per_second": 33.753,
-      "eval_steps_per_second": 2.225,
+      "eval_accuracy": 0.945054945054945,
+      "eval_loss": 0.17991583049297333,
+      "eval_runtime": 2.7033,
+      "eval_samples_per_second": 33.663,
+      "eval_steps_per_second": 2.22,
       "step": 87
     },
     {
       "epoch": 12.413793103448276,
-      "grad_norm": 28.829450607299805,
-      "learning_rate": 2.707006369426752e-05,
-      "loss": 0.1291,
+      "grad_norm": 22.67920684814453,
+      "learning_rate": 1.984126984126984e-05,
+      "loss": 0.1979,
       "step": 90
     },
     {
       "epoch": 12.96551724137931,
-      "eval_accuracy": 0.8571428571428571,
-      "eval_loss": 0.35304272174835205,
-      "eval_runtime": 2.6568,
-      "eval_samples_per_second": 34.252,
-      "eval_steps_per_second": 2.258,
+      "eval_accuracy": 0.9120879120879121,
+      "eval_loss": 0.25460970401763916,
+      "eval_runtime": 2.605,
+      "eval_samples_per_second": 34.933,
+      "eval_steps_per_second": 2.303,
       "step": 94
     },
     {
       "epoch": 13.931034482758621,
-      "eval_accuracy": 0.8571428571428571,
-      "eval_loss": 0.3431708812713623,
-      "eval_runtime": 2.6667,
-      "eval_samples_per_second": 34.124,
-      "eval_steps_per_second": 2.25,
+      "eval_accuracy": 0.9010989010989011,
+      "eval_loss": 0.25756293535232544,
+      "eval_runtime": 2.6309,
+      "eval_samples_per_second": 34.589,
+      "eval_steps_per_second": 2.281,
       "step": 101
     },
     {
       "epoch": 14.482758620689655,
-      "grad_norm": 11.376550674438477,
-      "learning_rate": 2.229299363057325e-05,
-      "loss": 0.0836,
+      "grad_norm": 3.1801395416259766,
+      "learning_rate": 1.388888888888889e-05,
+      "loss": 0.0633,
       "step": 105
     },
     {
       "epoch": 14.89655172413793,
-      "eval_accuracy": 0.8901098901098901,
-      "eval_loss": 0.2557763159275055,
-      "eval_runtime": 2.6632,
-      "eval_samples_per_second": 34.169,
-      "eval_steps_per_second": 2.253,
+      "eval_accuracy": 0.9340659340659341,
+      "eval_loss": 0.1996360421180725,
+      "eval_runtime": 2.7759,
+      "eval_samples_per_second": 32.782,
+      "eval_steps_per_second": 2.161,
       "step": 108
     },
     {
       "epoch": 16.0,
-      "eval_accuracy": 0.8351648351648352,
-      "eval_loss": 0.45357275009155273,
-      "eval_runtime": 2.6883,
-      "eval_samples_per_second": 33.85,
-      "eval_steps_per_second": 2.232,
+      "eval_accuracy": 0.9560439560439561,
+      "eval_loss": 0.1824313998222351,
+      "eval_runtime": 2.6954,
+      "eval_samples_per_second": 33.762,
+      "eval_steps_per_second": 2.226,
       "step": 116
     },
     {
       "epoch": 16.551724137931036,
-      "grad_norm": 5.885956287384033,
-      "learning_rate": 1.751592356687898e-05,
-      "loss": 0.1053,
+      "grad_norm": 0.8367649912834167,
+      "learning_rate": 7.936507936507936e-06,
+      "loss": 0.0311,
       "step": 120
     },
     {
       "epoch": 16.96551724137931,
-      "eval_accuracy": 0.8461538461538461,
-      "eval_loss": 0.46378985047340393,
-      "eval_runtime": 2.6068,
-      "eval_samples_per_second": 34.909,
-      "eval_steps_per_second": 2.302,
+      "eval_accuracy": 0.9560439560439561,
+      "eval_loss": 0.18344764411449432,
+      "eval_runtime": 2.7125,
+      "eval_samples_per_second": 33.549,
+      "eval_steps_per_second": 2.212,
       "step": 123
     },
     {
       "epoch": 17.93103448275862,
-      "eval_accuracy": 0.8351648351648352,
-      "eval_loss": 0.5922185182571411,
-      "eval_runtime": 2.6604,
-      "eval_samples_per_second": 34.205,
-      "eval_steps_per_second": 2.255,
+      "eval_accuracy": 0.9230769230769231,
+      "eval_loss": 0.2769739627838135,
+      "eval_runtime": 2.693,
+      "eval_samples_per_second": 33.791,
+      "eval_steps_per_second": 2.228,
       "step": 130
     },
     {
       "epoch": 18.620689655172413,
-      "grad_norm": 5.540600299835205,
-      "learning_rate": 1.2738853503184714e-05,
-      "loss": 0.0734,
+      "grad_norm": 0.14054416120052338,
+      "learning_rate": 1.984126984126984e-06,
+      "loss": 0.0154,
       "step": 135
     },
     {
       "epoch": 18.896551724137932,
-      "eval_accuracy": 0.8681318681318682,
-      "eval_loss": 0.3474012613296509,
-      "eval_runtime": 2.7317,
-      "eval_samples_per_second": 33.313,
-      "eval_steps_per_second": 2.196,
+      "eval_accuracy": 0.945054945054945,
+      "eval_loss": 0.2113342434167862,
+      "eval_runtime": 2.672,
+      "eval_samples_per_second": 34.056,
+      "eval_steps_per_second": 2.245,
       "step": 137
     },
     {
-      "epoch": 20.0,
-      "eval_accuracy": 0.8461538461538461,
-      "eval_loss": 0.5510861277580261,
-      "eval_runtime": 2.6777,
-      "eval_samples_per_second": 33.984,
-      "eval_steps_per_second": 2.241,
-      "step": 145
-    },
-    {
-      "epoch": 20.689655172413794,
-      "grad_norm": 7.2109575271606445,
-      "learning_rate": 7.961783439490445e-06,
-      "loss": 0.046,
-      "step": 150
-    },
-    {
-      "epoch": 20.96551724137931,
-      "eval_accuracy": 0.8461538461538461,
-      "eval_loss": 0.5115231275558472,
-      "eval_runtime": 2.7397,
-      "eval_samples_per_second": 33.215,
-      "eval_steps_per_second": 2.19,
-      "step": 152
-    },
-    {
-      "epoch": 21.93103448275862,
-      "eval_accuracy": 0.9010989010989011,
-      "eval_loss": 0.3096972107887268,
-      "eval_runtime": 2.8257,
-      "eval_samples_per_second": 32.204,
-      "eval_steps_per_second": 2.123,
-      "step": 159
-    },
-    {
-      "epoch": 22.75862068965517,
-      "grad_norm": 5.06452751159668,
-      "learning_rate": 3.1847133757961785e-06,
-      "loss": 0.0139,
-      "step": 165
-    },
-    {
-      "epoch": 22.896551724137932,
-      "eval_accuracy": 0.9120879120879121,
-      "eval_loss": 0.2510547637939453,
-      "eval_runtime": 2.7875,
-      "eval_samples_per_second": 32.645,
-      "eval_steps_per_second": 2.152,
-      "step": 166
-    },
-    {
-      "epoch": 24.0,
-      "eval_accuracy": 0.9120879120879121,
-      "eval_loss": 0.28634488582611084,
-      "eval_runtime": 2.6795,
-      "eval_samples_per_second": 33.961,
-      "eval_steps_per_second": 2.239,
-      "step": 174
-    },
-    {
-      "epoch": 24.137931034482758,
-      "eval_accuracy": 0.9120879120879121,
-      "eval_loss": 0.28424757719039917,
-      "eval_runtime": 2.9131,
-      "eval_samples_per_second": 31.238,
-      "eval_steps_per_second": 2.06,
-      "step": 175
+      "epoch": 19.310344827586206,
+      "eval_accuracy": 0.9340659340659341,
+      "eval_loss": 0.20573703944683075,
+      "eval_runtime": 2.6636,
+      "eval_samples_per_second": 34.164,
+      "eval_steps_per_second": 2.253,
+      "step": 140
     },
     {
-      "epoch": 24.137931034482758,
-      "step": 175,
-      "total_flos": 8.67232581632557e+17,
-      "train_loss": 0.1937425845861435,
-      "train_runtime": 510.3099,
-      "train_samples_per_second": 22.29,
-      "train_steps_per_second": 0.343
+      "epoch": 19.310344827586206,
+      "step": 140,
+      "total_flos": 6.939281828084982e+17,
+      "train_loss": 0.17856683683182512,
+      "train_runtime": 404.7192,
+      "train_samples_per_second": 22.485,
+      "train_steps_per_second": 0.346
     }
   ],
   "logging_steps": 15,
-  "max_steps": 175,
+  "max_steps": 140,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 25,
+  "num_train_epochs": 20,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -337,7 +278,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8.67232581632557e+17,
+  "total_flos": 6.939281828084982e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea6a0c157d895bd61838d31585af25facef08d9f0026f770573053513da974f8
+oid sha256:a0044340807b243d2b01a87986689059b3e0318763b9a2d90e95e77c2c8a3768
 size 5176