Training in progress, step 500

ipu_config.json

@@ -1,12 +1,13 @@
 {
+  "auto_loss_scaling": false,
   "decompose_grad_sum": false,
-  "device_iterations": 2,
-  "embedding_serialization_factor": 4,
+  "device_iterations": 1,
+  "embedding_serialization_factor": 2,
   "enable_half_first_order_momentum": true,
   "enable_half_partials": true,
   "executable_cache_dir": "./exe_cache",
   "execute_encoder_on_cpu_for_generation": false,
-  "gradient_accumulation_steps": 64,
+  "gradient_accumulation_steps": 16,
   "inference_device_iterations": 5,
   "inference_replication_factor": 4,
   "ipus_per_replica": 4,
@@ -16,7 +17,12 @@
     4,
     4
   ],
-  "matmul_proportion": 0.25,
+  "matmul_proportion": [
+    0.25,
+    0.25,
+    0.25,
+    0.25
+  ],
   "optimizer_state_offchip": true,
   "optimum_version": "1.3.0",
   "output_mode": "final",

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc52b84cb95a5b6b6b240355ea46fc7d9e6554e531dbe267b566a047a2481499
-size 261514473
+oid sha256:1854dafd8849f3aac5d15e43a378d11f97efab3a84b6131e2aabceb48e4675fc
+size 162657115

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ee5f4f3b3d5de4e91c70cda2cbf0ccfdf8ca6bf37533031273cc536c8edc58d
-size 2607
+oid sha256:3a02d178f04ca1e4f706509a8c510c6ef45c0cdb20d4bd5e0ea009974c779701
+size 2671