Training in progress, step 70, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:042e0c3a918c14b727dcfc1ea925b2d3c7674a5622906b94464180e5e38cf6fe
+oid sha256:66f6660bd73672b3675091a0d67850217bcf7e44f530e1d6bb4f07c7bc2ae294
 size 8668296

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec58794edaff89de4b0e55040b97e67e16b429077cbe7d7491978d263b5897d8
+oid sha256:365a83e2b9244e8cf01476b8783cea8987932fb83387898b4d10f00e6f768206
 size 17405562

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4aea8230fc4c9acbc31d25294372a7c4056d3b0a0b7e5461bb93deed7a35475b
+oid sha256:23cc361cb1ce03d818b88d94dcf4157d4e7dfe0dd7c27d3f1c597b16cda9f983
 size 13990

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2ebe7f3850f9be52d6f93fc9f544af2e9ed508950333a07fde53b974c0f7a69
+oid sha256:e79f9d4e7265c13ec8562f770a4b1f0d931af3562e2beb00601dee97b33f105c
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 8.267716535433072,
   "best_model_checkpoint": "./exp/wft-test-model/checkpoint-60",
-  "epoch": 3.03,
+  "epoch": 3.13,
   "eval_steps": 10,
-  "global_step": 60,
+  "global_step": 70,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -480,6 +480,7 @@
     },
     {
       "epoch": 2.13,
+      "eval_pred": null,
       "grad_norm": 2.5712523460388184,
       "learning_rate": 0.00049,
       "loss": 0.4704,
@@ -487,6 +488,7 @@
     },
     {
       "epoch": 2.14,
+      "eval_pred": null,
       "grad_norm": 2.026331901550293,
       "learning_rate": 0.00048,
       "loss": 0.2717,
@@ -494,6 +496,7 @@
     },
     {
       "epoch": 2.15,
+      "eval_pred": null,
       "grad_norm": 1.7977256774902344,
       "learning_rate": 0.00047,
       "loss": 0.5672,
@@ -501,6 +504,7 @@
     },
     {
       "epoch": 2.16,
+      "eval_pred": null,
       "grad_norm": 2.375070810317993,
       "learning_rate": 0.00046,
       "loss": 0.4817,
@@ -508,6 +512,7 @@
     },
     {
       "epoch": 2.17,
+      "eval_pred": null,
       "grad_norm": 3.009258985519409,
       "learning_rate": 0.00045000000000000004,
       "loss": 0.533,
@@ -515,6 +520,7 @@
     },
     {
       "epoch": 2.18,
+      "eval_pred": null,
       "grad_norm": 2.3523738384246826,
       "learning_rate": 0.00044,
       "loss": 0.4127,
@@ -522,6 +528,7 @@
     },
     {
       "epoch": 2.19,
+      "eval_pred": null,
       "grad_norm": 5.932104110717773,
       "learning_rate": 0.00043,
       "loss": 0.3008,
@@ -529,6 +536,7 @@
     },
     {
       "epoch": 3.01,
+      "eval_pred": null,
       "grad_norm": 1.4854482412338257,
       "learning_rate": 0.00042,
       "loss": 0.3457,
@@ -536,6 +544,7 @@
     },
     {
       "epoch": 3.02,
+      "eval_pred": null,
       "grad_norm": 2.2406277656555176,
       "learning_rate": 0.00041,
       "loss": 0.3722,
@@ -543,6 +552,7 @@
     },
     {
       "epoch": 3.03,
+      "eval_pred": null,
       "grad_norm": 1.1445629596710205,
       "learning_rate": 0.0004,
       "loss": 0.2341,
@@ -561,6 +571,90 @@
       "eval_wer": 8.267716535433072,
       "eval_wer_time": 0.0059778690338134766,
       "step": 60
+    },
+    {
+      "epoch": 3.04,
+      "grad_norm": 1.4139453172683716,
+      "learning_rate": 0.00039000000000000005,
+      "loss": 0.2319,
+      "step": 61
+    },
+    {
+      "epoch": 3.05,
+      "grad_norm": 1.311193823814392,
+      "learning_rate": 0.00038,
+      "loss": 0.235,
+      "step": 62
+    },
+    {
+      "epoch": 3.06,
+      "grad_norm": 1.1820735931396484,
+      "learning_rate": 0.00037,
+      "loss": 0.1914,
+      "step": 63
+    },
+    {
+      "epoch": 3.07,
+      "grad_norm": 1.2226531505584717,
+      "learning_rate": 0.00035999999999999997,
+      "loss": 0.3147,
+      "step": 64
+    },
+    {
+      "epoch": 3.08,
+      "grad_norm": 1.2730989456176758,
+      "learning_rate": 0.00035,
+      "loss": 0.2505,
+      "step": 65
+    },
+    {
+      "epoch": 3.09,
+      "grad_norm": 1.0159199237823486,
+      "learning_rate": 0.00034,
+      "loss": 0.15,
+      "step": 66
+    },
+    {
+      "epoch": 3.1,
+      "grad_norm": 1.2265311479568481,
+      "learning_rate": 0.00033,
+      "loss": 0.1765,
+      "step": 67
+    },
+    {
+      "epoch": 3.11,
+      "grad_norm": 0.8005520105361938,
+      "learning_rate": 0.00032,
+      "loss": 0.0663,
+      "step": 68
+    },
+    {
+      "epoch": 3.12,
+      "grad_norm": 1.507002353668213,
+      "learning_rate": 0.00031,
+      "loss": 0.1685,
+      "step": 69
+    },
+    {
+      "epoch": 3.13,
+      "grad_norm": 1.3489922285079956,
+      "learning_rate": 0.0003,
+      "loss": 0.2233,
+      "step": 70
+    },
+    {
+      "epoch": 3.13,
+      "eval_cer": 83.52272727272727,
+      "eval_cer_time": 0.0032608509063720703,
+      "eval_decode_time": 0.5359196662902832,
+      "eval_loss": 0.20264121890068054,
+      "eval_pred": "| i | Label | Prediction |\n| --- | --- | --- |\n| 0 | MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL | MISTER QUILTER IS THE APOSTLE OF THE MDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPELLLTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH-\"-\"-\"TH-\"TH-\"-\"THTHTH-\"\"-\"-THTHTHTHTHTH\"-\"-\"-\"-\"-\"-\"-\"-\"-\"--\"\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"--\"\"-\"-\"--\"\"-=\"--\"-\"-\"\"-\"-\"--\"\"-\"- |\n| 1 | NOR IS MISTER QUILTER'S MANNER LESS INTERESTING THAN HIS MATTER | NOR IS MISTER QUILTER'S MANNER LESS INTERESTING THAN HIS MATTERMTHMMMMMTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH-\"-\"\"-\"-\"-TH\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 2 | HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LOOMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READILY TO THE MIND | HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LOOMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READILY TO THE MINDMMMTHMTHMMMMTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH%%THTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH%%%%%%%%%%%%%% |\n| 3 | HE HAS GRAVE DOUBTS WHETHER SIR FREDERICK LEIGHTON'S WORK IS REALLY GREEK AFTER ALL AND CAN DISCOVER IN IT BUT LITTLE OF ROCKY ITHACA | HE HAS GRAVE DBSTS WHETHER SIR FDERICK LITTON'S WORK IS REALLY GREE AF AFTER ALL AND CAN DISCOVER IN IT BUT LITTLE OF ROCKY ITHAKATHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH |\n| 4 | LINNELL'S PICTURES ARE A SORT OF UP GUARDS AND AT EM PAINTINGS AND MASON'S EXQUISITE IDYLLS ARE AS NATIONAL AS A JINGO POEM MISTER BIRKET FOSTER'S LANDSCAPES SMILE AT ONE MUCH IN THE SAME WAY THAT MISTER CARKER USED TO FLASH HIS TEETH AND MISTER JOHN COLLIER GIVES HIS SITTER A CHEERFUL SLAP ON THE BACK BEFORE HE SAYS LIKE A SHAMPOOER IN A TURKISH BATH NEXT MAN | LINNELL'S PICTURES ARE A SORT OF UP GUARDS AND AT EM PAINTINGS AND MASON'S EXQUISATE IDIDLLS ARE AS NATIONAL AS A JINGO POEM MISTER BIRKET FOSTER'S LANDSCAPES SMILE AT ONE MUCH IN THE SAME WAY THAT MISTER CARKER USED TO FLASH HIS TEETH AND MISTER JOHN COLLIER GIVES HIS SITTER A CHEERFUL SLAP ON THE BACK BEFORE HE SAYS LIKE A SHAMPOOER IN A TURKISH BATH NEXT MAN |\n| 5 | IT IS OBVIOUSLY UNNECESSARY FOR US TO POINT OUT HOW LUMINOUS THESE CRITICISMS ARE HOW DELICATE IN EXPRESSION | IT IS OBVIOUSLY UNNESSESSARY FOR US TO POINT OUT HOW LUMINUS THESE CRITICISMS ARE HOW DALIICATE IN EXPRESSIONTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\" |\n| 6 | ON THE GENERAL PRINCIPLES OF ART MISTER QUILTER WRITES WITH EQUAL LUCIDITY | ON THE GENERAL PRINCIPLES OF ART MISTER QUILTER WRITES WITH EQUAL LUCIDITYEEEEEETHETHETH \u266aD \u266aEDE======\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 7 | PAINTING HE TELLS US IS OF A DIFFERENT QUALITY TO MATHEMATICS AND FINISH IN ART IS ADDING MORE FACT | PAINTING HE TELLS US IS OF A DIFFERENT QUALITY TO MATHEMATICS AND FINISH IN ART IS ADDING MORE FACT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\"-\"-\"-\"-%%%\"-\"-\"-\"-\"-%\"-\"-%\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 8 | AS FOR ETCHINGS THEY ARE OF TWO KINDS BRITISH AND FOREIGN | AS FOR ETCHINGS THEY ARE OF TWO KINDS BRITISH AND FOREIGNBTHTHTHTHTHTHTHTHTH |\n| 9 | HE LAMENTS MOST BITTERLY THE DIVORCE THAT HAS BEEN MADE BETWEEN DECORATIVE ART AND WHAT WE USUALLY CALL PICTURES MAKES THE CUSTOMARY APPEAL TO THE LAST JUDGMENT AND REMINDS US THAT IN THE GREAT DAYS OF ART MICHAEL ANGELO WAS THE FURNISHING UPHOLSTERER | HE LAMENTS MOST BITTERLY THE DIVORCE THAT HAS BEEN MADE BETWEEN DECORATIVE ART AND WHAT WE USUALLY CALL PICTURES MAKES THE CUSTOMARY APPEAL TO THE LAST JGMENT AND REMINDS US THAT IN THE GREAT DAYS OF ART M ANGELO WAS THE FURNISHING UPHOLSTERER |\n",
+      "eval_runtime": 1.3431,
+      "eval_samples_per_second": 7.445,
+      "eval_steps_per_second": 1.489,
+      "eval_wer": 9.05511811023622,
+      "eval_wer_time": 0.006304025650024414,
+      "step": 70
     }
   ],
   "logging_steps": 1,
@@ -580,7 +674,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6406358999040000.0,
+  "total_flos": 7515685232640000.0,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null