Training in progress, step 50, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69250899be317f17eb6150755da5a0a35d49a63d155d515e801a858c946226c1
+oid sha256:24b449598bad47929b0ce7ce9703863717369e58b5e614b756d22f3d1ac8a9ba
 size 8668296

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc80bb61a5290e89e718be215e85f6cf4a9fd78360d6ee75b2bc094d230cb93e
+oid sha256:accaa4c683aa5e1fe5f340f3b9cbc03bfdc27beb6f4c087a84121f607359471d
 size 17405562

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:905434e45334ec4cba7c0ef0c068c1bd1f793066d9f7aa304f9fefb141203eb2
+oid sha256:7974626dae7c2fcda8d413a7af00ed25e0f770e9e83e99fdad74474c21304fba
 size 13990

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76ae49a69ba5bc0ce251dde9168db7d92fe29ee0c2ea6e44dcdcace2cd0e9412
+oid sha256:a225c467e7b8ca846ccba612e68b44c91016d285329de640f01561d0afab82dc
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 37.79527559055118,
-  "best_model_checkpoint": "./exp/wft-test-model/checkpoint-30",
-  "epoch": 2.02,
+  "best_metric": 16.535433070866144,
+  "best_model_checkpoint": "./exp/wft-test-model/checkpoint-50",
+  "epoch": 2.12,
   "eval_steps": 10,
-  "global_step": 40,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -292,6 +292,7 @@
     },
     {
       "epoch": 1.12,
+      "eval_pred": null,
       "grad_norm": 3.4594056606292725,
       "learning_rate": 0.00031,
       "loss": 0.8506,
@@ -299,6 +300,7 @@
     },
     {
       "epoch": 1.13,
+      "eval_pred": null,
       "grad_norm": 2.7810869216918945,
       "learning_rate": 0.00032,
       "loss": 0.6984,
@@ -306,6 +308,7 @@
     },
     {
       "epoch": 1.1400000000000001,
+      "eval_pred": null,
       "grad_norm": 3.656834840774536,
       "learning_rate": 0.00033,
       "loss": 0.7591,
@@ -313,6 +316,7 @@
     },
     {
       "epoch": 1.15,
+      "eval_pred": null,
       "grad_norm": 3.26741886138916,
       "learning_rate": 0.00034,
       "loss": 0.7747,
@@ -320,6 +324,7 @@
     },
     {
       "epoch": 1.16,
+      "eval_pred": null,
       "grad_norm": 2.717848777770996,
       "learning_rate": 0.00035,
       "loss": 0.7748,
@@ -327,6 +332,7 @@
     },
     {
       "epoch": 1.17,
+      "eval_pred": null,
       "grad_norm": 2.2561049461364746,
       "learning_rate": 0.00035999999999999997,
       "loss": 0.8973,
@@ -334,6 +340,7 @@
     },
     {
       "epoch": 1.18,
+      "eval_pred": null,
       "grad_norm": 3.305250406265259,
       "learning_rate": 0.00037,
       "loss": 0.8124,
@@ -341,6 +348,7 @@
     },
     {
       "epoch": 1.19,
+      "eval_pred": null,
       "grad_norm": 5.550938129425049,
       "learning_rate": 0.00038,
       "loss": 0.7307,
@@ -348,6 +356,7 @@
     },
     {
       "epoch": 2.01,
+      "eval_pred": null,
       "grad_norm": 2.3965165615081787,
       "learning_rate": 0.00039000000000000005,
       "loss": 0.5959,
@@ -355,6 +364,7 @@
     },
     {
       "epoch": 2.02,
+      "eval_pred": null,
       "grad_norm": 1.919407844543457,
       "learning_rate": 0.0004,
       "loss": 0.4517,
@@ -373,6 +383,90 @@
       "eval_wer": 40.94488188976378,
       "eval_wer_time": 0.02174687385559082,
       "step": 40
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 2.023576021194458,
+      "learning_rate": 0.00041,
+      "loss": 0.6744,
+      "step": 41
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 1.814568281173706,
+      "learning_rate": 0.00042,
+      "loss": 0.4547,
+      "step": 42
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 2.640611171722412,
+      "learning_rate": 0.00043,
+      "loss": 0.5251,
+      "step": 43
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 2.0394105911254883,
+      "learning_rate": 0.00044,
+      "loss": 0.561,
+      "step": 44
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 2.003967046737671,
+      "learning_rate": 0.00045000000000000004,
+      "loss": 0.4994,
+      "step": 45
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 2.668008804321289,
+      "learning_rate": 0.00046,
+      "loss": 0.51,
+      "step": 46
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 2.123786687850952,
+      "learning_rate": 0.00047,
+      "loss": 0.6328,
+      "step": 47
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 2.4257912635803223,
+      "learning_rate": 0.00048,
+      "loss": 0.3986,
+      "step": 48
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 2.094670057296753,
+      "learning_rate": 0.00049,
+      "loss": 0.4531,
+      "step": 49
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 2.141101837158203,
+      "learning_rate": 0.0005,
+      "loss": 0.4352,
+      "step": 50
+    },
+    {
+      "epoch": 2.12,
+      "eval_cer": 88.13920454545455,
+      "eval_cer_time": 0.0032167434692382812,
+      "eval_decode_time": 0.5202732086181641,
+      "eval_loss": 0.4492875337600708,
+      "eval_pred": "| i | Label | Prediction |\n| --- | --- | --- |\n| 0 | MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL | MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPELBBBBBBTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH\"-\"-\"-TH\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 1 | NOR IS MISTER QUILTER'S MANNER LESS INTERESTING THAN HIS MATTER | NOR IS MISTER QUILTER'S MANNER LESS INTERESTING THAN HIS MATTERSSMMMTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 2 | HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LOOMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READILY TO THE MIND | HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LUMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READDILY TO THE MINDSSSSSSSSSTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH##THTHTHTHTH#TH###THTHTHTH |\n| 3 | HE HAS GRAVE DOUBTS WHETHER SIR FREDERICK LEIGHTON'S WORK IS REALLY GREEK AFTER ALL AND CAN DISCOVER IN IT BUT LITTLE OF ROCKY ITHACA | HE HAS GRAVE DOTSTS WHETHER SIR FREDIC LIGHTTON'S WORK IS REALLY GREEK AFTER ALL AND CAN DISCOVER IN IT BUT LITTLE OF ROCKY ITHAKAAAAEEEEEEEEETHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH |\n| 4 | LINNELL'S PICTURES ARE A SORT OF UP GUARDS AND AT EM PAINTINGS AND MASON'S EXQUISITE IDYLLS ARE AS NATIONAL AS A JINGO POEM MISTER BIRKET FOSTER'S LANDSCAPES SMILE AT ONE MUCH IN THE SAME WAY THAT MISTER CARKER USED TO FLASH HIS TEETH AND MISTER JOHN COLLIER GIVES HIS SITTER A CHEERFUL SLAP ON THE BACK BEFORE HE SAYS LIKE A SHAMPOOER IN A TURKISH BATH NEXT MAN | LINELELL'S PICTURES ARE A SORT OF UP GUARDS AND ATEM PAINTINGS AND MAS'S EXQUISATE ITOLLESS ARE AS NATIONAL AS A JINGO POEM MISTER BIRKET FOSTER'S LANDSAPES SMILE AT ONE MUCH IN THE SAME WAY THAT MISTER CARCER USED TO FLASH HIS TEETH AND MISTER JOHN COLLIER GIVES HIS SITTER A CHIRFUL SLAP ON THE BACK BEFORE HE SAYS LIKE A SHAMPOOER IN A TURKISH BATH NEXT MAN |\n| 5 | IT IS OBVIOUSLY UNNECESSARY FOR US TO POINT OUT HOW LUMINOUS THESE CRITICISMS ARE HOW DELICATE IN EXPRESSION | HE IS OVIOUSLY UNNESSESSARY FOR US TO POINT OUT HOW LUMENUS THESE CRITICISMS ARE HOW DEELICATE IN EXPRESSIONENENENENEN))THTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHCOMTH-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"\"--\"-\" |\n| 6 | ON THE GENERAL PRINCIPLES OF ART MISTER QUILTER WRITES WITH EQUAL LUCIDITY | ON THE GENERAL PRINCEIPLES OF ART MISTER QUILTER RIGHTITES WITH EQUIL LUCIDITYSEEEEEEEEEEEDEEEEEEEDDDED\"-D\"-\"-D++\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"- |\n| 7 | PAINTING HE TELLS US IS OF A DIFFERENT QUALITY TO MATHEMATICS AND FINISH IN ART IS ADDING MORE FACT | PINTING HE TELLS US IS OF A DIFFERENT QUALITY TO MATHEMATICS AND FINISH IN ART IS ADDING MORE FACTTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH++%++++++++++++++++++++++++++++++++++++++\"-+++++\"-\"-+\"-+\"-\"-++\"-\"-\"-\"-\"-++++ |\n| 8 | AS FOR ETCHINGS THEY ARE OF TWO KINDS BRITISH AND FOREIGN | AS FOR ATCHINGS THEY ARE OF TWO KINDS BRITISH AND FOREIGNTHTHTHTHTH |\n| 9 | HE LAMENTS MOST BITTERLY THE DIVORCE THAT HAS BEEN MADE BETWEEN DECORATIVE ART AND WHAT WE USUALLY CALL PICTURES MAKES THE CUSTOMARY APPEAL TO THE LAST JUDGMENT AND REMINDS US THAT IN THE GREAT DAYS OF ART MICHAEL ANGELO WAS THE FURNISHING UPHOLSTERER | HE LAMENTS MOST BITTERLY THE DEIVORCE THAT HAS BEEN MADE BEWEEN DEECRATIVE ART AND WHAT WE USEALLY CALL PICTURES MAKEES A CUSTOMARY APPEAL TO THE LAST JGENT AND REMINES US THAT IN THE GREAT DAYS OF ART M ANGELO WAS THE FURNISHING APHOLSTTERER |\n",
+      "eval_runtime": 1.3178,
+      "eval_samples_per_second": 7.588,
+      "eval_steps_per_second": 1.518,
+      "eval_wer": 16.535433070866144,
+      "eval_wer_time": 0.005387067794799805,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -392,7 +486,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4270905999360000.0,
+  "total_flos": 5380232232960000.0,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null