Training in progress, step 40, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05665b3b0b7410dbac32ea4039a959ce9cffcc1e0bae00b0da1c2117044d3ea6
+oid sha256:69250899be317f17eb6150755da5a0a35d49a63d155d515e801a858c946226c1
 size 8668296

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e8aa59d612822872cc18d650c63b9330f76f41b76702632f04dc2641d2c4e88
+oid sha256:cc80bb61a5290e89e718be215e85f6cf4a9fd78360d6ee75b2bc094d230cb93e
 size 17405562

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47cfa69a42c93dbca65569ec81b1155b1d488146e32a5777f7ff34a66ab6b1e6
+oid sha256:905434e45334ec4cba7c0ef0c068c1bd1f793066d9f7aa304f9fefb141203eb2
 size 13990

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15152ad07ee530f37d1795684ca0f2da347a19398aeb2c8d3d75fa4e1eaeb789
+oid sha256:76ae49a69ba5bc0ce251dde9168db7d92fe29ee0c2ea6e44dcdcace2cd0e9412
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 37.79527559055118,
   "best_model_checkpoint": "./exp/wft-test-model/checkpoint-30",
-  "epoch": 1.11,
+  "epoch": 2.02,
   "eval_steps": 10,
-  "global_step": 30,
+  "global_step": 40,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,7 @@
     },
     {
       "epoch": 1.02,
+      "eval_pred": null,
       "grad_norm": 4.428985118865967,
       "learning_rate": 0.00021,
       "loss": 1.2975,
@@ -205,6 +206,7 @@
     },
     {
       "epoch": 1.03,
+      "eval_pred": null,
       "grad_norm": 4.232193470001221,
       "learning_rate": 0.00022,
       "loss": 1.3636,
@@ -212,6 +214,7 @@
     },
     {
       "epoch": 1.04,
+      "eval_pred": null,
       "grad_norm": 5.8515167236328125,
       "learning_rate": 0.00023,
       "loss": 1.3856,
@@ -219,6 +222,7 @@
     },
     {
       "epoch": 1.05,
+      "eval_pred": null,
       "grad_norm": 3.324622869491577,
       "learning_rate": 0.00024,
       "loss": 0.9259,
@@ -226,6 +230,7 @@
     },
     {
       "epoch": 1.06,
+      "eval_pred": null,
       "grad_norm": 5.164034366607666,
       "learning_rate": 0.00025,
       "loss": 1.1038,
@@ -233,6 +238,7 @@
     },
     {
       "epoch": 1.07,
+      "eval_pred": null,
       "grad_norm": 8.681169509887695,
       "learning_rate": 0.00026000000000000003,
       "loss": 1.4779,
@@ -240,6 +246,7 @@
     },
     {
       "epoch": 1.08,
+      "eval_pred": null,
       "grad_norm": 3.7555525302886963,
       "learning_rate": 0.00027,
       "loss": 1.0174,
@@ -247,6 +254,7 @@
     },
     {
       "epoch": 1.09,
+      "eval_pred": null,
       "grad_norm": 4.5491042137146,
       "learning_rate": 0.00028000000000000003,
       "loss": 1.1516,
@@ -254,6 +262,7 @@
     },
     {
       "epoch": 1.1,
+      "eval_pred": null,
       "grad_norm": 3.3533918857574463,
       "learning_rate": 0.00029,
       "loss": 0.7413,
@@ -261,6 +270,7 @@
     },
     {
       "epoch": 1.11,
+      "eval_pred": null,
       "grad_norm": 3.5604212284088135,
       "learning_rate": 0.0003,
       "loss": 0.8667,
@@ -279,6 +289,90 @@
       "eval_wer": 37.79527559055118,
       "eval_wer_time": 0.03960013389587402,
       "step": 30
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 3.4594056606292725,
+      "learning_rate": 0.00031,
+      "loss": 0.8506,
+      "step": 31
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 2.7810869216918945,
+      "learning_rate": 0.00032,
+      "loss": 0.6984,
+      "step": 32
+    },
+    {
+      "epoch": 1.1400000000000001,
+      "grad_norm": 3.656834840774536,
+      "learning_rate": 0.00033,
+      "loss": 0.7591,
+      "step": 33
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 3.26741886138916,
+      "learning_rate": 0.00034,
+      "loss": 0.7747,
+      "step": 34
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 2.717848777770996,
+      "learning_rate": 0.00035,
+      "loss": 0.7748,
+      "step": 35
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 2.2561049461364746,
+      "learning_rate": 0.00035999999999999997,
+      "loss": 0.8973,
+      "step": 36
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 3.305250406265259,
+      "learning_rate": 0.00037,
+      "loss": 0.8124,
+      "step": 37
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 5.550938129425049,
+      "learning_rate": 0.00038,
+      "loss": 0.7307,
+      "step": 38
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 2.3965165615081787,
+      "learning_rate": 0.00039000000000000005,
+      "loss": 0.5959,
+      "step": 39
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 1.919407844543457,
+      "learning_rate": 0.0004,
+      "loss": 0.4517,
+      "step": 40
+    },
+    {
+      "epoch": 2.02,
+      "eval_cer": 73.79261363636364,
+      "eval_cer_time": 0.002995014190673828,
+      "eval_decode_time": 0.5137369632720947,
+      "eval_loss": 0.6337034106254578,
+      "eval_pred": "| i | Label | Prediction |\n| --- | --- | --- |\n| 0 | MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL | MISTER QUILTER IS THE APOSLE OF THE MDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPLETHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH#THTHTHTH# \u266a# -####TH# - - -THTHTH#\"- \u266aTHTHTH#THTH#\"-\"-##++++++\"-\"-\"-++\"-\"-\"-\"-\"-\"-\"-\"-++\"-\"-\"-\"-\"-\"-\"-\"-+\"-\"-\"-\"-\"-\"-\"-\"-+\"- |\n| 1 | NOR IS MISTER QUILTER'S MANNER LESS INTERESTING THAN HIS MATTER | NOR IS MrISTER QUILTERSS MANNER LESS INTERESTING THAN HIS MATTERMM)) - -)))) - - - - -### - - - - - -\"-\"- - - -\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-++\"-\"-\"-\"-\"-\"- |\n| 2 | HE TELLS US THAT AT THIS FESTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEF LOOMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCCUR MOST READILY TO THE MIND | HE TELLS US THAT AT THIS FESTTIVE SEASON OF THE YEAR WITH CHRISTMAS AND ROAST BEEEF LUMING BEFORE US SIMILES DRAWN FROM EATING AND ITS RESULTS OCRUR MOST READDILY TO THE MINDGTHTHG%%%###%####%####%####%###%%###%####%### |\n| 3 | HE HAS GRAVE DOUBTS WHETHER SIR FREDERICK LEIGHTON'S WORK IS REALLY GREEK AFTER ALL AND CAN DISCOVER IN IT BUT LITTLE OF ROCKY ITHACA | HE HAS GRAVE DOTSTS WHETHER SERIR FRDIC LITTON'SS WORK IS REALLY GREK AFTER ALL AND CAN DISCOVER IN IT BUT LITTLE OF ROCKY ITHAKAGTHTHTHGTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH##GTHTHTH#TH####THTHTH##TH##THTHTHTH |\n| 4 | LINNELL'S PICTURES ARE A SORT OF UP GUARDS AND AT EM PAINTINGS AND MASON'S EXQUISITE IDYLLS ARE AS NATIONAL AS A JINGO POEM MISTER BIRKET FOSTER'S LANDSCAPES SMILE AT ONE MUCH IN THE SAME WAY THAT MISTER CARKER USED TO FLASH HIS TEETH AND MISTER JOHN COLLIER GIVES HIS SITTER A CHEERFUL SLAP ON THE BACK BEFORE HE SAYS LIKE A SHAMPOOER IN A TURKISH BATH NEXT MAN | LINELEL'S PICTURES ARE A SORT OF UPGARDS AND ADT PINTINGS AND MASSS EXQUISIT ITOLLEIS ARE AS NATIONAL AS A GINGO PEM MrISTER BRIRKET FOSTER'S LANDSAPES SMILE AT ONE MUCH IN THE SAME WAY THAT MISTER CARCER USED TO FASH HIS TEETH AND MISTER John COLLIER GIVES HIS CITTER A CHURFUL SLAP IN THE BACK BEFORE HE SAYS LIKE A SHAMPURER AND A TURKISH BATH NEXT MAN |\n| 5 | IT IS OBVIOUSLY UNNECESSARY FOR US TO POINT OUT HOW LUMINOUS THESE CRITICISMS ARE HOW DELICATE IN EXPRESSION | IT IS OBVIOUSLY UNNESSESSARY FOR US TO POINT OUT HOW LUMINUS THESE CRITICISMS ARE HOW DEELICATE IN EXPRESSIONTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTHTH-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"\"-\"--\"-\"-\"-\"\"--\"\"-\"-\"-\"--\" |\n| 6 | ON THE GENERAL PRINCIPLES OF ART MISTER QUILTER WRITES WITH EQUAL LUCIDITY | ON THE GENERAL PRINCEIPLES OF ART MISTER QUILTER RITES WITH EQUIL LUCIDITYEEE - -EEEEEEE+ -++++++++++++\"-++++++\"-\"-\"-+++\"-\"-\"-\"-+\"-+\"-++\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-\"-e\"-\"-\"-\"-\"-\"-e\"-\"-ee\"-\"-\"-\"-\"-\"-\"-ee\"-\"- |\n| 7 | PAINTING HE TELLS US IS OF A DIFFERENT QUALITY TO MATHEMATICS AND FINISH IN ART IS ADDING MORE FACT | PINING HE TELLS US IS OF A DIFFERENT QUALITY TO MHEMATICS AND FINISH IN ART IS ADDING MORE FACT%%%%%%%%%%%%%%%%#%%%%%%++%%+++++++++++++++++++++++++++++++++++++++++++++\"-++++++++\"-+++++++ |\n| 8 | AS FOR ETCHINGS THEY ARE OF TWO KINDS BRITISH AND FOREIGN | AS FOR ECHINGS THEY ARE OF TWO KINDS BRITISH AND FOREIGN \u266a \u266a) \u266a \u266a \u266a \u266a \u266a \u266a \u266a \u266a \u266a \u266a \u266a \u266a \u266a \u266a \u266aTH |\n| 9 | HE LAMENTS MOST BITTERLY THE DIVORCE THAT HAS BEEN MADE BETWEEN DECORATIVE ART AND WHAT WE USUALLY CALL PICTURES MAKES THE CUSTOMARY APPEAL TO THE LAST JUDGMENT AND REMINDS US THAT IN THE GREAT DAYS OF ART MICHAEL ANGELO WAS THE FURNISHING UPHOLSTERER | HE LAMENTS MOST BITTERLY THE DEIVORCE THAT HAS BEEN MADE BEWEEN DEECRATIVE ART AND WHAT WE USELALLY CALL PICTURES MAKES A CUSTOMER APPEAL TO THE LAST JGENT AND REMINES US THAT IN THE GREAT DAYS OF ART M ANGELO WAS THE FURNISHING APHOLSTTERER |\n",
+      "eval_runtime": 1.3876,
+      "eval_samples_per_second": 7.207,
+      "eval_steps_per_second": 1.441,
+      "eval_wer": 40.94488188976378,
+      "eval_wer_time": 0.02174687385559082,
+      "step": 40
     }
   ],
   "logging_steps": 1,
@@ -298,7 +392,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3244779233280000.0,
+  "total_flos": 4270905999360000.0,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null