Training in progress, step 1500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cdda364a7dea7ba94515761ffb2ace46580132bb59bff458e83e48f877aff5a
+oid sha256:8a6231b9a43188b3030be892c139d7894a62c3fea99c8c6c7eb17b00365a57d7
 size 36730224

last-checkpoint/ar_diffusion_info.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9f18b53100851e8533c053a9a05241ff32f7709a9624b367b683d0de95176f9
+oid sha256:61e1d1dea84b6c812c4ddd82f2f7f4ea581df5126382ca1bf3a48d5758288836
 size 1544

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56779e621a4c332b005c80c0b7213bbef6847256028ea00b31ba836b8203e11e
+oid sha256:923a3b0ad55df1dc931abd14ea79f0c2ae7566e46df76993edc14d667615816a
 size 73588346

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d00cc8448e3ffb4901f6ba77807b5c278ba2af145c1beeacecc1ec2179e63f6
-size 14244
+oid sha256:42de46b2e4c5179833d46fe28b80f469c3d40255a80ec3a104ef7ad1fd3d8210
+size 14180

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08e8e3095f52bd6edeff23718f3d5712489fc946ab10f7e28fb30a543dc6baa8
+oid sha256:3adce9cc9139db0a326056a6714b81b8c58d21b77098b37dae23cf33f822452c
 size 988

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ca71486ea7bff6a3b5fb792928d9943140a9a2fdb3589f5685c55d472ab8413
+oid sha256:f03648c25873beacb3afe9bdbd6315225ba0d09c0be3adf22a9ad7dc42cafe47
 size 1064

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 500,
-  "best_metric": 5.163902759552002,
-  "best_model_checkpoint": "./ar-diffusion-checkpoints-fixed/checkpoint-500",
-  "epoch": 0.038458580109222366,
+  "best_global_step": 1250,
+  "best_metric": 4.76948356628418,
+  "best_model_checkpoint": "./ar-diffusion-checkpoints-fixed/checkpoint-1000",
+  "epoch": 0.1153757403276671,
   "eval_steps": 250,
-  "global_step": 500,
+  "global_step": 1500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -94,6 +94,178 @@
       "eval_samples_per_second": 50.725,
       "eval_steps_per_second": 12.681,
       "step": 500
+    },
+    {
+      "epoch": 0.0423044381201446,
+      "grad_norm": 6.368561744689941,
+      "learning_rate": 0.0001997714463808015,
+      "loss": 5.0723,
+      "step": 550
+    },
+    {
+      "epoch": 0.04615029613106684,
+      "grad_norm": 4.666464328765869,
+      "learning_rate": 0.00019951172635898504,
+      "loss": 5.0754,
+      "step": 600
+    },
+    {
+      "epoch": 0.04999615414198908,
+      "grad_norm": 9.395013809204102,
+      "learning_rate": 0.00019925200633716855,
+      "loss": 4.7718,
+      "step": 650
+    },
+    {
+      "epoch": 0.05384201215291132,
+      "grad_norm": 4.279386520385742,
+      "learning_rate": 0.00019899228631535204,
+      "loss": 4.781,
+      "step": 700
+    },
+    {
+      "epoch": 0.05768787016383355,
+      "grad_norm": 4.274901866912842,
+      "learning_rate": 0.00019873256629353558,
+      "loss": 4.9529,
+      "step": 750
+    },
+    {
+      "epoch": 0.05768787016383355,
+      "eval_loss": 4.844481945037842,
+      "eval_runtime": 18.9171,
+      "eval_samples_per_second": 52.862,
+      "eval_steps_per_second": 13.216,
+      "step": 750
+    },
+    {
+      "epoch": 0.06153372817475579,
+      "grad_norm": 3.241445541381836,
+      "learning_rate": 0.0001984728462717191,
+      "loss": 5.0586,
+      "step": 800
+    },
+    {
+      "epoch": 0.06537958618567802,
+      "grad_norm": 5.880845546722412,
+      "learning_rate": 0.00019821312624990263,
+      "loss": 4.952,
+      "step": 850
+    },
+    {
+      "epoch": 0.06922544419660026,
+      "grad_norm": 3.81998610496521,
+      "learning_rate": 0.00019795340622808614,
+      "loss": 5.0012,
+      "step": 900
+    },
+    {
+      "epoch": 0.0730713022075225,
+      "grad_norm": 4.52741003036499,
+      "learning_rate": 0.00019769368620626966,
+      "loss": 4.7775,
+      "step": 950
+    },
+    {
+      "epoch": 0.07691716021844473,
+      "grad_norm": 27.68866729736328,
+      "learning_rate": 0.00019743916058488948,
+      "loss": 4.9852,
+      "step": 1000
+    },
+    {
+      "epoch": 0.07691716021844473,
+      "eval_loss": 4.836514472961426,
+      "eval_runtime": 19.7103,
+      "eval_samples_per_second": 50.735,
+      "eval_steps_per_second": 12.684,
+      "step": 1000
+    },
+    {
+      "epoch": 0.08076301822936698,
+      "grad_norm": 5.79191255569458,
+      "learning_rate": 0.00019717944056307302,
+      "loss": 4.6731,
+      "step": 1050
+    },
+    {
+      "epoch": 0.0846088762402892,
+      "grad_norm": 4.957877159118652,
+      "learning_rate": 0.00019691972054125653,
+      "loss": 4.7966,
+      "step": 1100
+    },
+    {
+      "epoch": 0.08845473425121145,
+      "grad_norm": 3.2968597412109375,
+      "learning_rate": 0.00019666000051944005,
+      "loss": 4.752,
+      "step": 1150
+    },
+    {
+      "epoch": 0.09230059226213368,
+      "grad_norm": 6.059363842010498,
+      "learning_rate": 0.00019640028049762359,
+      "loss": 4.7368,
+      "step": 1200
+    },
+    {
+      "epoch": 0.09614645027305592,
+      "grad_norm": 5.9793171882629395,
+      "learning_rate": 0.0001961405604758071,
+      "loss": 4.9613,
+      "step": 1250
+    },
+    {
+      "epoch": 0.09614645027305592,
+      "eval_loss": 4.76948356628418,
+      "eval_runtime": 18.6703,
+      "eval_samples_per_second": 53.561,
+      "eval_steps_per_second": 13.39,
+      "step": 1250
+    },
+    {
+      "epoch": 0.09999230828397816,
+      "grad_norm": 7.604544162750244,
+      "learning_rate": 0.0001958808404539906,
+      "loss": 4.8365,
+      "step": 1300
+    },
+    {
+      "epoch": 0.10383816629490039,
+      "grad_norm": 6.4756083488464355,
+      "learning_rate": 0.00019562112043217412,
+      "loss": 4.8381,
+      "step": 1350
+    },
+    {
+      "epoch": 0.10768402430582263,
+      "grad_norm": 3.605341672897339,
+      "learning_rate": 0.00019536140041035764,
+      "loss": 4.7004,
+      "step": 1400
+    },
+    {
+      "epoch": 0.11152988231674486,
+      "grad_norm": 3.4805853366851807,
+      "learning_rate": 0.00019510168038854115,
+      "loss": 4.7293,
+      "step": 1450
+    },
+    {
+      "epoch": 0.1153757403276671,
+      "grad_norm": 4.5123796463012695,
+      "learning_rate": 0.0001948419603667247,
+      "loss": 4.8315,
+      "step": 1500
+    },
+    {
+      "epoch": 0.1153757403276671,
+      "eval_loss": 4.784451961517334,
+      "eval_runtime": 18.5259,
+      "eval_samples_per_second": 53.979,
+      "eval_steps_per_second": 13.495,
+      "step": 1500
     }
   ],
   "logging_steps": 50,