diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/latest_checkpoint.txt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/latest_checkpoint.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3a20e0a2317452a5fef87d4ce85f82dd599975be
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/latest_checkpoint.txt
@@ -0,0 +1 @@
+iter_000003200.pt
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10d6fd592e9f46aba42e39c7957644db15a589e0
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d29e7f9d68a31ec5def21758f4e7c7e1b0280d1ea663176de8fcd35b9a1abd5c
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d5977e2b2638d6e1d7f8a0f4791d486f0ee832ea
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d287ed13bc9d5f5009798da158a3436474e28e03c2d17767b38202733cc5660
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63d721b0b473eaa1342b596589e4c5249b10b064
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79236f1b5f8d66e41b9faf6cdc0346cedcd3b98b8fe9736ac545a969967782c2
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7eb0a1893897cb5be4ba6a25c73bc0e300590736
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02b0c78dcebd7c2fcabf0666f99c5eb894ed23bb3d27b59b450826e51c3e3d76
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af879fd6b038c9fa3344f5461684c5aee7003dd9
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65c14e40bf84f120191532682756306560af930a45a683670a4eed9bba9f2acf
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..81170fc62e8ba96ad12cc35ceb04dfd8a30ce8fd
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:544e198c26d6270425624d86d713c53549a526620c793d958d485242f3a7e4db
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dabec5f6a5e6d2017067581bde45d1c4fa901629
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c513093aea4e3eb6737eaae700de84f1e68db1919da2b16a5c42b7930ec1f04e
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e45cc817e46b0e029433089cdccbd82f14182b03
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ecce11ef144b8c44664f8ab6729872dbb494fb96ef7985229d8d3e9159ec7f1
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a1dd7126d03cf42ceeb8c36b154ffcae48308fc4
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54be1b8c377cc65fae108471c648a892a51895db6c899e51a274b08fe63bea2d
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fb600ad670a29784b41c4504375e672011071378
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:440264d0526a9545c63ebb011943377f767076e252f6bb649d625daa23827c24
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1ec17acb354fc6da86716d4bd3a0889a7bfeb22
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38407916fe71dda530a31ae174eb0aad0a5193a47a0b2fa679bf1c61b82bf832
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c34e049081e0252dafbfcd95da0b3e6ae4954e0
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cc727f6e3b37b5dd5e3c03a057ada7657c1b5e1d7d5ed71e94e947712cafe18
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df7e5a7e9f61e0318d2b3efe40497e248aed46fd
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cb59911a4ee4ce3d26793fdd688b68268bd38a1a17a22999e75e3818200e491
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78ef070eefdbd18391e3c05269377630e7be1fbc
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b80a035eb2236cb28c4888e7390cbfc84661a3ddf0f5b975d51e91490fd9c001
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..881f463a73b87c845dd716c10da6355a42756e09
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6c458fff7e7f8c22aa8ba367872aca25be78c54d428819c5766a61a8e192b8b
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc083a7abea022e245f81d8b9b959b9e1b0091d0
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:011a2cc17ed1969cb9131696e5cc95534382a68b45bb58f4b9654a584ea8e01c
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9acffe615f07d8ea551a7f9083ffc95e5f2bebc
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:027dfc917e51a48487b8be2f9e481be41dfb4c60ef7c05dac448930bec5a9079
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aae6164cabd6c8b7b6e264dfa06cdc4f08d94b5f
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98d1c991328d7c13b1993116fd2ee4cfec76cd8b94325909b742f7a453591788
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74161959760bfc60f8da5651cc868e72cd08850c
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2baa9253eba3b77b9e481be6b5d29ee0e14644c02aed90cbd8211874fefebaf1
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..101408416aa4811dde310d5f8c8f163fac4cdf5f
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87631ecfe2cad7b5b8056e03ad07f5546d63c0618dbd33cc0f79b9465ff9ef2c
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cf1839367bc38db7700a4842bac893d202f182fe
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0278f6d08024d1013645980ecb9b52026c55f1cf0396fa9c23c5f64487cef5fe
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9843b3192806692f7a057b83835e3430b9193a07
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9448209e009051648ee0a377a63cf13911aed8fe4b8dc63aa443ec395cb1bea3
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3702d1baa8e1e86467f0c7ec27bf1f29455e4029
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d375b3025564b58603266d1d4a23beb5fa7885cc77f7f76fe3fbef869a8d52c7
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d1f987de187bb2feac75ac80bda75dd32194e6f
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44376e978524da139a7ea7d1ae851a3c1288f0f6ffaf0eb474bfb400f96fbb6d
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d9ecc4285b92f9288b0530eafb09a735ddcd957a
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3fd4bd7a248639de2c87a5c914d7e9825cff548f361788b8de8e01158bc0c9f
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e9e257a7dbaee0601ca017b1faa60fbb85b7246
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9e65979cdbd446149bc8fdc48493cad2c8f53ac8fe1a2982ad9530cd02584b4
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9cfb4926a4d495c2e147011386c924435b1a49e
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bad603dcc446532d459ebfb44228f8e1af304213dc84f8d97cfb80b3c2f39e46
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b087c4c135ddfb69e468f992924f5f7316bbf3cd
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:482cb942c58011ff2add8dc4eb1b78a8c26070a86589cb41279a1b860beba000
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5cdcdbb31a774530718dd4d83857c92d20d03ae4
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9ccfca92ad3e4f16a17c64e62c44e85fef880d1c9b7b39257d8c459a94b1736
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7b3ed0b40150eadca7117a19edd6039e9cfa63cf
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:861e3f7c56e6617c97ee4fb4d2a1fc33005932f38721ec9915a0b8ef5017c59c
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d6b81a2404bd15b5a338059832afbaa27a2d8238
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f777c39dfb3a7b9d4831fdf889eafe05e958937213c92e6295c2a92bfbc89c48
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51d624170cd944b08c7d95275429d362133634c3
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbd6e06dd249523841bf2ee2e2e344f00f268537ce06819e5f43006a5a643b5d
+size 161817668
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bdd7bb55821805562833d0cbcafa508573364097
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd849c193cf48012705d240cfce6db64ce8ebe87b5922db9333f96cf077a0b00
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b0f7faf408f0b5f76ed969a77edfdb1a4a1ec328
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:873bd003a99086b901bf5a700df815b41662730f9d111f0d49821da0ed036429
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..debb017a0b7bbe8274b0b0fb10f6c48eb2374427
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2776da3267dfb4f1e149ba23cefcf9b6d6d87f824711ef543f128c27e1cb5e74
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..31c8672235512e069d53c3c2dc11f72e64234598
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b419292947a28a4770740402c047dafa854c0362e594fd89463461dd09c7ffb3
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a71ae4bfbd15cf223fb1e87146fb060dc1e4fbd
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0dbbc234b63345b6624418b14b3bb9e1a3568578d6dcf029a44c3ec2082043e
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dfbdf5dfcbae928b60f08bd2833c30fba11786db
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdf36856d66173f96b215c454b4133b5635e528fa4cc115ab0c661a454ec17ac
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c96271f8464e910b94cadde5d716bf5ab8f5cec6
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30987701353982f502f68142bbf2d8d2bdc0de16e8cddde8bac14a8bd86d2041
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..efeb40ad5d0e7bee54a6fdd9eba4931b090a1ac7
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a85289a633869f771d203ffd1a461793988607f6e3f47ce8cd8a20d767a878a2
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e40faea7e5311b13065eb4e89cf366b6081ac87b
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b5b878639fcc745d8a0e0677688ee1c96747a8e62d02a6934ce0c1f79ea36f1
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..848163ccbbcf0807333ae609dd6f045abdeced19
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2c4c484995d250af86fd0ada44f48654967b5eccc2a79f450895646ed214d55
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..994978a8c4c613c1a5793a812f64fa8819e48f5b
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92aeec6f1d197e33bb3a32fe5465212af8c533c4098ec492e4e87add1c56c628
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..104e4941496aacb2b9a9b3c9954efcd800189dbb
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fcbed75bc7f305fac0d5108b050bf95f677ccf143c6a81de1ee5c32e4c213fa
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e21297d69f6ecc603a60be44157f89df2fd8deed
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3ab69e7942fd6bbad862887905db49362cd15e2a67858ed4a3a9476e875b6c
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d7c10e2de363fd061a70552e65b5c7b249e062e
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bea20466e3ca2a67a6c88fdb730d3a0c49b6488402a2c30e3bd13e88ddcacc16
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b2394e13a85fc8f333215090b456c5e4110e3ab5
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c68ce641323a97c025f7f9f85df94629278d53d50489b27472540198a270a83
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cdf50bab06539712df3e9a7d2f6cf2a3ed47ccff
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c78b4f167e966aad4af97c9689f5f0f62d9209f5a0da3aaa02bf5be458b8f42a
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0699aa5bdf44e3beabc14aeb086b95b0813930cc
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f51c5b23efd139715d82cb266220b95d3fb6e96df72f910db6f975896442490
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd45b4bd014fb873369ba7ab461af2db667eff08
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:455d25cfb281d9613ae5b14a4284f25879119c7f833b14420bb034c7d6704ab8
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c783e90df8eb822e77f925d807c7abadd27422c
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58eeecd66e656920fbd3e503691377650ea558d583b341eedb27099aeb611cd3
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3f2deca4a0d824f87b2574e4ecb22c27e9a4073b
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd79b9f043faf2161fc9721bdeb8b1064d41edbb8b0e7807c2840273d75ca635
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3c006432b517c7246a45bf29d820b175900ad901
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b583f492b2abda7122a388f9ba2f19bcdc0c9749afca20ced9bcfaa324c58687
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..929ce69e3bd3b97ceff3ffe27dfb4a475fcea659
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45c5ba6f394170020c3ee576a31f4bb199f3fd75e02c49a116739754af8bfeb1
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96b142a0e2a5d2640a9fbd0bb4a9e3fafba1f844
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c21d39bf514b3d14eb100f1e6a341f451e4ac88b0cfe84fd43676aec947a794
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2fff3cc11b5f8a910383988bf424211f9dc60cb9
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e37325eaae7130bcb153d0ce3312589514b19b730dc529437e97ed611fad2a9
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..771db40720e37ca2bf4643463c2cc68dc918e9e4
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:747b9b10316d17bc0c33013c8ced15f2a3a04442489a70d8092698a104827805
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2e975eb50f88c756af354db7b7f7ab72b12118b2
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f861815632260c0a6c302c6df9599069aeb7b2485fc296b18e76331408e4183d
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e799761d1ae36e1f1c6ebedd77f0fd54d3731fa2
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1da13d10e76c75151bc3e13e7d7c981e28acb972c91b1cfcce92d105152dceef
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..43adb5c4211bcac6975bcf973d1e3eda64b01305
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc45eaba2793038fa78a6a50c4209b27eecb58cf7bffce47a5df34d46e62002b
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..317609216a1dacf1511ecf781acbeab9ea062e29
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d805739c60696948a51ebab4659ebbdd128d198007cc0d1a0afa499c3b1906cc
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4c3c342f1f6028bff76d1c603bc3e2273016174
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7737d3db4d9af2565e597ec5d5dada6f701ac64e5484b535d2cd85a0b938bdd0
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..87af43c8e512e089746fc34157cc25423f36c719
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c1b45360b3c6ac63955ad2426fb7f16ca9a8a4c785c309da2eb6bbd932e38bc
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbb00fe1186821ad293e525b05996b7fa27408b4
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52d8a4838755c3ea26aad30ce6ab0bdf755d2c640001a2905c0fc91661169012
+size 647025631
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7876c3fff7b6d32f8d6ad5e443478b663f6488fa
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f693badcdbd793f02e3acedcb949a3b09bdcf5a33aa3f0634850a03f8600401
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51e13fbfe1715932a8e95c68a4d647cb020bc348
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83fd51848d73f84af93bd0a90e468618a3b6f84c6b5855c253ea3c706a52dd42
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1bc15bc3d95b2fd15d84d4c67e3f822766ad6f
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99b46fea48fdcc51a58c73da124dc6f6cc13155cd3873116255cc2c9b121a425
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8c185e6b7a0af0266e4f43e8ef6c8fd536838761
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5dc213dcd4fffac941b7fea96d25099f23f8b52a30bb875ec875b6f08cdd18a
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33e4d9f39d65f0a4156f2d4466b28c4c96b8f0b3
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f39221ddd84d035772c4fd8a6ef8d1d95902c2e4f6809ae251491d99a6a300d8
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..731a9075cb19471264bcf13cb76f42df94b5f673
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f071560ea0ca662afeb043d2d1eabcb18854f5ba8a1a38520ef46e747f8b788
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..205b251fff0d5cf15aef72f1db9b0844f5c6e22e
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bef6c88b860ac719bbc9d545267a923d3f8e0633f6febe29e37fe1afcb6f3acc
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68ef83b966e3494281b39fe0a6bd06726a1bffa0
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d39b3c7fb1d91e6139144e2e065e824effd04ed4e304062ca571df66ac41f87a
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e97dc5c737c8908e704e414d816b0ef9c26053dc
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:132398a4dc3ee2b05556c55161a2e98e607afca0372336ca3d2bf8b1eb571d97
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a80b9ec1fb4573a8169f220ce29adcc176e2b316
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47a6bae9e822d2753aee6b0b62d6f98bb46ade9ec4e41da26490e0201d09c172
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba37fcebf2a5d2fb00e7e1e203041892f7af06cf
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8ccb4cd0692af13b54f44b82fd68a5e84267f3c5f14480352a39a5fe5ab6ef1
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba68d013a09b98be941b4c1b9054eab21a31ce06
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cae38d0753eb6c498617acae582d9613739959e8a63b4e07eafefcbf40f64e7a
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d43fe60ad22ec97ed48610d1de7503d139082b8
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf3b185e6174b4e224fa63991f6fb6736ffecb94aa9a744984cfabe7944ec9b6
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..210863b994d31b6e2d1a8ec9313b7e84326276fa
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98814442e7bbcea116bc8268889b7bafaf1ba0d989182305c15d818924ccec1
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4831b45e092b3906edc0a051341277d7c19e7e74
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89bed9ad82a1395e0d7beb51d50936d57897bfe338e9d47b02d5dc7896145f8c
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b081794edaa664c8db7ca0ba67807f878a6eb359
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94bcb346ef58274d17d7b4f0e69002ddf055bbb8d29fa9381aaddd98764ab885
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7522c35cf5d752f55fdc02e825735f455346349
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:396e592e0f5ebdea7cbfd7c653839eb04de17bc355a49332021aa40060f59aca
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b36dd8486c20381fd65d8ab4d880d1facbfb9b99
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22917846837e99079048b759e8b3d3b72c696ff7b6d5b30d574ef8c5ffc913cc
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2919b898acd26914fd1621290c244bcdf7d5db10
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:136fe15895d079508f5dca5603eeaf9b94ffac4093ea48db4ad5a8b710df21f2
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..da14483f7c167e09e1bc713d74f2789945f877b5
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca84df360cf96ab61e97368db93153f62818e5e02cde160fd6b3ee325627dbf2
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88f2d2d921f684a37f4a9a49ddb5caf341553dd2
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b4b8bee5f4da5963270979c9e2ae5420f081a8c324cf2ffffd87eabf2bbdbeb
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3502bb5a441e97c36501800c848cb2d4fe3229d1
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cb5f2c8d53f425811d1c3c90ef884305f18036d386b173d05cbb38c29756478
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fee13cccbe59d5005394393056107df994a8fc2
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:460aaa47371ae02bfaed9e6daaf32e2252fae53593975d048d40a44f561e4a9b
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..03b9799bf12786235dcbf6e3d561854eacc42203
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78d3ec3e3146e57668749565c8cba5f7017ee5ad5228b0f53d845f864f23a28c
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..11f181f0d4902697595b9515d27bf9ed51fb590a
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcad57dd9cfe6bce0578b585c09eb16af22226fac7a24fb1dc757cc3b58825bc
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e3f51b2fde45371fe2fba8ae09edafb59d02d843
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a32eae3139178444d45f31c0e66c3e50e66696f427428a349a892db20349afd
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bc2629aa863bc209fbde393dd0f84c224bd9f15f
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23ba088bdd90c9d86ca1afbb9fbebe6f0632154b2a61ed8c21272b34b58c4210
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..63aca46eab99c70231b6fe045ca53fa2714eab54
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:632bbe86e9f7ec6b01d30c2ecf1753084c34f0276540f0ead544c40804d58231
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..907643b28ffc98d3c85a45fac1a0fdfd2b385292
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9062725b423df3f23162ab9e780b37eece740d2f9b1d0340ab832ab5b27f2d25
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3b3d020984ffc22c39ad67b7ecb63a127b048969
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d89fe280eaa9da2d9da6d2de29515dfd6e4c038da1bea27bbeb4ef7f3c0beebd
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..faff1bfcfe987636ade6e67b9e1453a72d3f1171
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46a385c3eff988d5c5d3ccaa50308a2219a8efc7f16137668c59b96c3d814d21
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d300e14a8551623a4f216f050dc2713a1b3c297d
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d22f956a813777c48503a87c58c8aa41fb135a5617d10446cac16b5e6f2cfe5a
+size 1602
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5535bc0b7e67ce40094bbcc144c6bba3d7f8fe8b
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d950795f379aef8f4dff7386679d77b4a65ca4291bed09afced61b39d3dfa49f
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..813312ba9fab3ba8936aacc34d410dd62c4765be
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b63e3d448feca7bea909fef7514faae3cbe7ada857a5e211a6e46fdf3d005d22
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3f845ce292621fbaed110272c9ea22d422efc3b
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0f362aa11769c1a4c5261f596d15c1d4dd509a0eb98fbded2add580ad97b56c
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3241ce9c9934f0aef3cebd5590452542a3cea57b
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3962658429705a75a1c67650f778c2937060352be8d407ae95199f003fb716e8
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06e7e7a7fabd9aae34dcc411a98e243673da6aba
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0907f99ff4186b13c44c403126adb5aad8058db62ed10d75282f43f93a2c79e2
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..25e46db2f826f4123a73274622b4c4da10a1ed87
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d4241c8657960fc56d56e303b2f10176953d10a3c4c475826e14070d4f889bd
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ac8d636c1f69e400fb2345fad440b234660b95c
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e26f75721f3de13910989e0eb962a70dd1edef863de223aa0db6552d280f319d
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a69a9e4af0499dfba383e5ac67119b875919c5a1
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43055657c24bc47b81aa766f91dd9cad9277dea8fd5320ee339159e4477d5e4a
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0689425df7d0c06fd5776ab73c0faa5c463013a8
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf3af3f1095f92752b6396497ccccfef3f15a363cebb395bb892d595d7442321
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7ef193c3f24df75cfeae0fb0294adc34f9fba663
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a8eb54f575f232d4f03d7c9594005cb06ae9ffb4bc249977d1154493cfbf75
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..993a77eb84366d11b24ffd6679d2ab4232a1e6cc
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72f4dc142705e94743fbb00be363e3befd9b4642adcf1511bcdfa5493786293b
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea1e43f74308025b7312d6c453f8fbdd8571f395
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65a22edf306f247f5b60230b033492a02be88bb51cbb55585d535431d75d67ba
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef9f260d17a17f9e5e48485bab77c0b73f91b656
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6d19ade99b7fdec1a42c107b3df22a6fe8945fc407843972b83551262df4306
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c093c6283274c160db4f2563a5f316dc79fd5c7
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c92d3f6150f9e3c0c060f9adbe711a75d70e7ece5ae839e8d639475ce762955
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c163fbcf6a21aa9bb44b6443b65f5798f6f24f48
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7053c4ac2524bdbf97217455f074cf8d8f747463838eda2b9ef2ca7541c455dd
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94990cba717425006db847b385f1fa967429d64e
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:064e3e842533ed6f4fbbbf831111751c87bfe938b153b7c635cdb2b7c9a59540
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2243c42de8f8ac59c8ceeba205f7d0794a11ae65
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d90aea88abdb41139f0b9594b7d450840341d8915f6895f9052883942b2242bd
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..210414a5372d430bdfd271328772b731ba389694
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:190ee185eb7fdff804e5fe1ddf9ce00ff9920e5e2bb5d4755cb96932ff1bed82
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68b643639f970becf5b9c584fb8289fff84b728f
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11b277ed1490891bc346b0c53c6aed9e5bb73e4313a13cf65981abe9f225c70f
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7da9e5bd053ec51c9b60c4d2c1bde2ab8a21f684
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2b3774816544e78e539e7d51e64b43a30c795a2947d1fe5a7dbd70c48e434f7
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68b6ab3f2eef3e53e7071a53593b63488a444e1b
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1149581fc64ad3bb27b55c1f262129c99510dc49e609d8394a04cfad5cb2479d
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2dc3be7903b2eafa6241c1f9fc6fc052e9f92266
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66a6708b0dedfc90c1d322f3b5ce321cf9c4ea23e9deaf2216846b9bbcdae305
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002300.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002300.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5bf9d5e7593843b4a93f1a05ad6e033c77f418fa
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002300.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c1853ed0a2c8b2676f84be03c6e46205adbe24c639ba097f15226ffcf6a4bfb
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002400.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002400.pt
new file mode 100644
index 0000000000000000000000000000000000000000..65132dd9b16822614e25385fbdbb40c8183fe20e
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002400.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebd3ca4055c7d21f21f4ac41393ba855e7099df1f555bc39de59329362209266
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002500.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a4f7709c2dd04934ec90f96b79fdbada73e1cddb
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd35cc182e37a966ed9d46be9743babb2d35b34ff1128c02314225150d874e6d
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002600.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002600.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45bc0713d850347794d4cbf5cb831dddaacef2bf
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002600.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:203211fd55e4dac526b25214e309d82ec44cff54e71220fcb27558b2e660bb2d
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002700.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002700.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd3e5ce4338634e8c1cdab7d44d81b5a07a9a3e1
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002700.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c13a7631e478408cb0aabc3f4fd46f3e5dabdf1441f567b2a9b8ccf7e64ac4cf
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002800.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002800.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5e09886fde71bcbfea37d2f03181cd96b04f60c9
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002800.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79aa7bdfc0f80b4403af5982910e067cd15e7aff148cb689fbd0852a5869e378
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002900.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002900.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a24fde815b5aa44fa2492b1f055a378546c2955c
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002900.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25fe7bedaac46c04672cbc2480a0313949e04be630cd83acbc1591f7ac10aa49
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003000.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f1e2b5d040b9275a63468816ee4950cd74cbd735
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:213554861dcd84b2c90083647dbafccb254edb3e8d462405116548b16bd548b6
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003100.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003100.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d9e10c3596269d9a34c7963d52916bfa9a7b76d5
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003100.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6812f8163328a6483e453c38606378d97f125ae3b4e291359b8eb988fe9e2952
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003200.pt b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003200.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b3a76cc523bd18196b12c7b09a6d260b85fe7a2
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003200.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:938692c47b0a8f2f7726865c37a01337b225258f42a448e18c17a132c262f4ca
+size 892
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/config.pkl b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/config.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..7ff1bff76479a49f613828565f8f5dca9bc7c2c4
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/config.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a6a036b0b0e618da8f2d786b0403a28f16e8d57a8e72a80ec5117d927963d70
+size 76324
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/config.yaml b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9f235ddc4ec22cbbf95dc7715ec515c706e3c66a
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/config.yaml
@@ -0,0 +1,531 @@
+checkpoint:
+  broadcast_via_filesystem: 'False'
+  dcp_allow_mismatched_size: 'False'
+  dcp_async_mode_enabled: 'False'
+  jit:
+    device: cuda
+    dtype: bfloat16
+    enabled: 'False'
+    input_shape: null
+    strict: 'True'
+  keys_not_to_resume: []
+  load_ema_to_reg: 'False'
+  load_path: ''
+  load_training_state: 'False'
+  only_load_scheduler_state: 'False'
+  save_iter: '100'
+  strict_resume: 'True'
+  type:
+    _target_: <class 'cosmos_predict2.checkpointer.Checkpointer'>
+    callbacks: null
+  verbose: 'True'
+data_config:
+  _target_: <function get_data_config at 0x70f0db8c5fc0>
+  config_name: so101_lerobot
+dataloader_train:
+  _target_: <class 'torch.utils.data.dataloader.DataLoader'>
+  batch_sampler: null
+  batch_size: '2'
+  collate_fn: null
+  dataset:
+    _target_: <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>
+    action_image_horizon: '56'
+    action_key: action
+    action_lowdim_horizon: '90'
+    action_shift_s: '0.2'
+    cache_dir: null
+    image_key: observation.images.front
+    image_resize:
+    - '480'
+    - '640'
+    language_embeddings_path: null
+    lowdim_target_fps: '30.0'
+    num_val_episodes: '1'
+    obs_image_horizon: '5'
+    repo_id: push-that-thing/task_1
+    root: null
+    seed: '42'
+    state_key: observation.state
+    t5_encoder_ckpt: null
+    target_fps: '5.0'
+    train: 'True'
+    val_ratio: '0.05'
+    video_backend: pyav
+  drop_last: 'True'
+  generator: null
+  in_order: 'True'
+  multiprocessing_context: null
+  num_workers: '8'
+  persistent_workers: 'True'
+  pin_memory: 'True'
+  pin_memory_device: ''
+  prefetch_factor: '8'
+  sampler:
+    _target_: <class 'cosmos_predict2.data.resumable_sampler.ResumableDistributedSampler'>
+    dataset:
+      _target_: <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>
+      action_image_horizon: '56'
+      action_key: action
+      action_lowdim_horizon: '90'
+      action_shift_s: '0.2'
+      cache_dir: null
+      image_key: observation.images.front
+      image_resize:
+      - '480'
+      - '640'
+      language_embeddings_path: null
+      lowdim_target_fps: '30.0'
+      num_val_episodes: '1'
+      obs_image_horizon: '5'
+      repo_id: push-that-thing/task_1
+      root: null
+      seed: '42'
+      state_key: observation.state
+      t5_encoder_ckpt: null
+      target_fps: '5.0'
+      train: 'True'
+      val_ratio: null
+      video_backend: pyav
+    num_replicas:
+      _target_: <function get_data_parallel_world_size at 0x70f33ef7d510>
+      partial_data_parallel: 'False'
+      with_context_parallel: 'False'
+    rank:
+      _target_: <function get_data_parallel_rank at 0x70f33ef7d630>
+      partial_data_parallel: 'False'
+      with_context_parallel: 'False'
+    seed: '0'
+    shuffle: 'True'
+  shuffle: null
+  timeout: '0'
+  worker_init_fn: null
+dataloader_val:
+  _target_: <class 'torch.utils.data.dataloader.DataLoader'>
+  batch_sampler: null
+  batch_size: '1'
+  collate_fn: null
+  dataset:
+    _target_: <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>
+    action_image_horizon: '56'
+    action_key: action
+    action_lowdim_horizon: '90'
+    action_shift_s: '0.2'
+    cache_dir: null
+    image_key: observation.images.front
+    image_resize:
+    - '480'
+    - '640'
+    language_embeddings_path: null
+    lowdim_target_fps: '30.0'
+    num_val_episodes: '1'
+    obs_image_horizon: '5'
+    repo_id: push-that-thing/task_1
+    root: null
+    seed: '42'
+    state_key: observation.state
+    t5_encoder_ckpt: null
+    target_fps: '5.0'
+    train: 'False'
+    val_ratio: '0.05'
+    video_backend: pyav
+  drop_last: 'False'
+  generator: null
+  in_order: 'True'
+  multiprocessing_context: null
+  num_workers: '0'
+  persistent_workers: 'False'
+  pin_memory: 'False'
+  pin_memory_device: ''
+  prefetch_factor: null
+  sampler:
+    _target_: <class 'cosmos_predict2.data.resumable_sampler.ResumableDistributedSampler'>
+    dataset:
+      _target_: <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>
+      action_image_horizon: '56'
+      action_key: action
+      action_lowdim_horizon: '90'
+      action_shift_s: '0.2'
+      cache_dir: null
+      image_key: observation.images.front
+      image_resize:
+      - '480'
+      - '640'
+      language_embeddings_path: null
+      lowdim_target_fps: '30.0'
+      num_val_episodes: '1'
+      obs_image_horizon: '5'
+      repo_id: push-that-thing/task_1
+      root: null
+      seed: '42'
+      state_key: observation.state
+      t5_encoder_ckpt: null
+      target_fps: '5.0'
+      train: 'False'
+      val_ratio: null
+      video_backend: pyav
+    num_replicas:
+      _target_: <function get_data_parallel_world_size at 0x70f33ef7d510>
+      partial_data_parallel: 'False'
+      with_context_parallel: 'False'
+    rank:
+      _target_: <function get_data_parallel_rank at 0x70f33ef7d630>
+      partial_data_parallel: 'False'
+      with_context_parallel: 'False'
+    seed: '0'
+    shuffle: 'False'
+  shuffle: null
+  timeout: '0'
+  worker_init_fn: null
+defaults:
+- _self_
+- data_config: null
+- video_dataset_train: null
+- video_dataset_val: null
+- dataloader_train: null
+- dataloader_val: null
+- world2action_pipe: null
+- optimizer: fusedadamw
+- scheduler: constant
+- model: null
+- callbacks:
+  - basic
+- net: null
+- ema: null
+- checkpoint: null
+- ckpt_type: null
+- experiment: null
+job:
+  group: so101
+  name: w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5
+  project: vam
+model:
+  _target_: <class 'cosmos_predict2.models.world2action_model.World2ActionModel'>
+  config:
+    _target_: cosmos_predict2.models.world2action_model.World2ActionModelConfig
+    action_dit_path: ''
+    data_config:
+      _target_: <function get_data_config at 0x70f0db8c5fc0>
+      config_name: so101_lerobot
+    ema:
+      enabled: false
+      iteration_shift: 0
+      rate: 0.1
+    fsdp_shard_size: '0'
+    init_lora_weights: 'True'
+    lora_alpha: '16'
+    lora_rank: '16'
+    lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2
+    loss_reduce: mean
+    loss_scale: '10.0'
+    pipe_config:
+      _target_: cosmos_predict2.configs.config_world2action.World2ActionPipelineConfig
+      ema:
+        enabled: false
+        iteration_shift: 0
+        rate: 0.1
+      net:
+        _target_: <class 'cosmos_predict2.models.world2action_dit.World2ActionDIT'>
+        adaln_lora_dim: '128'
+        atten_backend: flash_attn_no_cp
+        crossattn_emb_channels: '2048'
+        in_channels: '6'
+        max_horizon: '91'
+        mlp_ratio: '4.0'
+        model_channels: '512'
+        num_blocks: '12'
+        num_heads: '8'
+        out_channels: '6'
+        pair_timestep_feature_rank: '512'
+        sac_config:
+          every_n_blocks: 1
+          mode: none
+        use_adaln_lora: 'True'
+      precision: bfloat16
+      scheduler:
+        alpha: 1.0
+        beta: 1.0
+        num_denoising_steps: 10
+      xattn_layer_idx: '20'
+    precision: bfloat16
+    train_architecture: base
+    validation_mode: loss_only
+    video_dit_path: /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt
+    video_pipe_config:
+      adjust_video_noise: true
+      conditioner:
+        _target_: <class 'cosmos_predict2.conditioner.VideoConditioner'>
+        fps:
+          _target_: <class 'cosmos_predict2.conditioner.ReMapkey'>
+          dropout_rate: '0.0'
+          dtype: null
+          input_key: fps
+          output_key: fps
+        padding_mask:
+          _target_: <class 'cosmos_predict2.conditioner.ReMapkey'>
+          dropout_rate: '0.0'
+          dtype: null
+          input_key: padding_mask
+          output_key: padding_mask
+        text:
+          _target_: <class 'cosmos_predict2.conditioner.TextAttr'>
+          dropout_rate: '0.0'
+          input_key:
+          - obs/language_embedding
+        use_video_condition:
+          _target_: <class 'cosmos_predict2.conditioner.BooleanFlag'>
+          dropout_rate: '0.0'
+          input_key: fps
+          output_key: use_video_condition
+      conditioning_strategy: frame_replace
+      ema:
+        _target_: cosmos_predict2.configs.defaults.ema.EMAConfig
+        enabled: 'False'
+        iteration_shift: '0'
+        rate: '0.1'
+      guardrail_config:
+        checkpoint_dir: /home/nvidia/pdt-mimic/mimic-video/model/checkpoints
+        enabled: false
+        offload_model_to_cpu: true
+      input_image_key: images
+      input_video_key: video
+      max_num_conditional_frames: 2
+      min_num_conditional_frames: 1
+      net:
+        _target_: <class 'cosmos_predict2.models.video2world_dit.MinimalV1LVGDiT'>
+        adaln_lora_dim: '256'
+        atten_backend: minimal_a2a
+        concat_padding_mask: 'True'
+        extra_per_block_abs_pos_emb: 'False'
+        in_channels: '16'
+        max_frames: '128'
+        max_img_h: '240'
+        max_img_w: '240'
+        model_channels: '2048'
+        num_blocks: '28'
+        num_heads: '16'
+        out_channels: '16'
+        patch_spatial: '2'
+        patch_temporal: '1'
+        pos_emb_cls: rope3d
+        pos_emb_interpolation: crop
+        pos_emb_learnable: 'True'
+        rope_enable_fps_modulation: 'False'
+        rope_h_extrapolation_ratio: '3.0'
+        rope_t_extrapolation_ratio: '1.0'
+        rope_w_extrapolation_ratio: '3.0'
+        sac_config:
+          _target_: cosmos_predict2.models.text2image_dit.SACConfig
+          every_n_blocks: '1'
+          mode: predict2_2b_720
+        use_adaln_lora: 'True'
+      precision: bfloat16
+      rectified_flow_loss_weight_uniform: true
+      rectified_flow_t_scaling_factor: 1.0
+      resize_online: false
+      resolution: '480'
+      sigma_conditional: 0.0001
+      sigma_data: 1.0
+      state_ch: 16
+      state_t: 16
+      text_encoder:
+        cls: !!python/object/apply:imaginaire.constants.TextEncoderClass
+        - t5
+        t5:
+          ckpt_path: /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/text_encoder/t5-11b
+          embed_dim: 1024
+          num_tokens: 512
+      timestamps:
+        is_forward: false
+        nfe: 35
+        order: 7.0
+        t_max: 80.0
+        t_min: 0.002
+      tokenizer:
+        _target_: <class 'cosmos_predict2.tokenizers.tokenizer.TokenizerInterface'>
+        chunk_duration: '81'
+        load_mean_std: 'False'
+        name: tokenizer
+        temporal_window: '16'
+        vae_pth: /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth
+model_parallel:
+  _cpu_offloading_context: null
+  async_tensor_model_parallel_allreduce: false
+  autocast_dtype: torch.float32
+  barrier_with_L1_time: true
+  batch_p2p_comm: true
+  batch_p2p_sync: true
+  bf16: false
+  context_parallel_size: 1
+  cpu_offloading: false
+  cpu_offloading_activations: false
+  cpu_offloading_num_layers: 0
+  cpu_offloading_weights: false
+  cross_entropy_fusion_impl: native
+  cross_entropy_loss_fusion: false
+  deallocate_pipeline_outputs: false
+  defer_embedding_wgrad_compute: false
+  deterministic_mode: false
+  enable_autocast: false
+  expert_model_parallel_size: 1
+  expert_tensor_parallel_size: 1
+  finalize_model_grads_func: null
+  fp16: false
+  grad_scale_func: null
+  grad_sync_func: null
+  gradient_accumulation_fusion: false
+  hierarchical_context_parallel_sizes: null
+  microbatch_group_size_per_vp_stage: 1
+  moe_extended_tp: false
+  no_sync_func: null
+  num_microbatches_with_partial_activation_checkpoints: null
+  overlap_p2p_comm: false
+  overlap_p2p_comm_warmup_flush: false
+  param_sync_func: null
+  params_dtype: torch.float32
+  perform_initialization: true
+  pipeline_dtype: null
+  pipeline_model_parallel_comm_backend: null
+  pipeline_model_parallel_size: 1
+  pipeline_model_parallel_split_rank: null
+  sequence_parallel: false
+  tensor_model_parallel_size: 1
+  timers: null
+  tp_comm_atomic_ag: false
+  tp_comm_atomic_rs: false
+  tp_comm_bootstrap_backend: nccl
+  tp_comm_bulk_dgrad: true
+  tp_comm_bulk_wgrad: true
+  tp_comm_overlap: false
+  tp_comm_overlap_ag: true
+  tp_comm_overlap_disable_fc1: false
+  tp_comm_overlap_disable_qkv: false
+  tp_comm_overlap_rs: true
+  tp_comm_overlap_rs_dgrad: false
+  tp_comm_split_ag: true
+  tp_comm_split_rs: true
+  use_cpu_initialization: false
+  use_ring_exchange_p2p: false
+  use_te_rng_tracker: false
+  variable_seq_lengths: false
+  virtual_pipeline_model_parallel_size: null
+  wgrad_deferral_limit: 0
+optimizer:
+  _target_: <function get_base_optimizer at 0x70f0c93b6050>
+  betas:
+  - '0.9'
+  - '0.99'
+  capturable: 'True'
+  eps: 1e-08
+  lr: '9.999999999999999e-05'
+  master_weights: 'True'
+  model: null
+  optim_type: fusedadam
+  weight_decay: '0.1'
+scheduler:
+  _target_: <class 'cosmos_predict2.functional.lr_scheduler.LambdaLinearScheduler'>
+  cycle_lengths:
+  - '500000'
+  f_max:
+  - '1'
+  f_min:
+  - '0.2'
+  f_start:
+  - 1e-06
+  verbosity_interval: '0'
+  warm_up_steps:
+  - '1000'
+trainer:
+  callbacks:
+    device_monitor:
+      _target_: <class 'cosmos_predict2.callbacks.device_monitor.DeviceMonitor'>
+      every_n: '1000'
+      log_memory_detail: 'True'
+      step_size: '1'
+    ema:
+      _target_: <class 'imaginaire.utils.callback.EMAModelCallback'>
+      config: null
+      trainer: null
+    grad_clip:
+      _target_: <class 'cosmos_predict2.callbacks.grad_clip.GradClip'>
+      clip_norm: '10.0'
+      force_finite: 'True'
+      log_wandb: 'False'
+    iter_speed:
+      _target_: <class 'cosmos_predict2.callbacks.iter_speed.IterSpeed'>
+      every_n: '1000'
+      hit_thres: '5'
+    low_prec:
+      _target_: <class 'imaginaire.utils.callback.LowPrecisionCallback'>
+      config: null
+      trainer: null
+      update_iter: '1'
+    manual_gc:
+      _target_: <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>
+      every_n: '5'
+      warm_up: '5'
+    progress_bar:
+      _target_: <class 'imaginaire.utils.callback.ProgressBarCallback'>
+      config: null
+      trainer: null
+    wandb:
+      _target_: <class 'cosmos_predict2.callbacks.wandb_logger.WandBLogger'>
+      mode: null
+  cudnn:
+    benchmark: 'True'
+    deterministic: 'False'
+  ddp:
+    broadcast_buffers: 'True'
+    find_unused_parameters: 'False'
+    static_graph: 'True'
+  distributed_parallelism: ddp
+  grad_accum_iter: '8'
+  grad_scaler_args:
+    enabled: 'False'
+  logging_iter: '1000'
+  max_iter: '8000'
+  max_val_iter: '16'
+  memory_format: torch.preserve_format
+  profiling:
+    enable_memory_snapshot: 'False'
+    enable_profiling: 'False'
+    first_n_rank: '4'
+    profile_freq: '1'
+    profile_memory: 'True'
+    record_shape: 'True'
+    with_modules: 'True'
+    with_stack: 'True'
+  run_validation: 'True'
+  seed: '0'
+  timeout_period: '999999999'
+  type: <class 'imaginaire.trainer.ImaginaireTrainer'>
+  validation_iter: '100'
+video_dataset_train: null
+video_dataset_val: null
+world2action_pipe:
+  _target_: cosmos_predict2.configs.config_world2action.World2ActionPipelineConfig
+  ema:
+    enabled: false
+    iteration_shift: 0
+    rate: 0.1
+  net:
+    _target_: <class 'cosmos_predict2.models.world2action_dit.World2ActionDIT'>
+    adaln_lora_dim: '128'
+    atten_backend: flash_attn_no_cp
+    crossattn_emb_channels: '2048'
+    in_channels: '6'
+    max_horizon: '91'
+    mlp_ratio: '4.0'
+    model_channels: '512'
+    num_blocks: '12'
+    num_heads: '8'
+    out_channels: '6'
+    pair_timestep_feature_rank: '512'
+    sac_config:
+      every_n_blocks: 1
+      mode: none
+    use_adaln_lora: 'True'
+  precision: bfloat16
+  scheduler:
+    alpha: 1.0
+    beta: 1.0
+    num_denoising_steps: 10
diff --git a/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/stdout.log b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/stdout.log
new file mode 100644
index 0000000000000000000000000000000000000000..9ea9c5060e7e1998c37c97ab05becab4ffc0c093
--- /dev/null
+++ b/action_decoder/chain/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/stdout.log
@@ -0,0 +1,608 @@
+[05-15 15:55:19|INFO|imaginaire/trainer.py:116:__init__] Config:
+* model: {'config': {'validation_mode': 'full', 'train_architecture': 'base', 'lora_rank': 16, 'lora_alpha': 16, 'lora_target_modules': 'q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2', 'init_lora_weights': True, 'precision': 'bfloat16', 'loss_reduce': 'mean', 'loss_scale': 10.0, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, 'action_dit_path': '', 'video_dit_path': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_that_thing.pt', 'pipe_config': {'precision': 'bfloat16', 'scheduler': {'alpha': 1.0, 'beta': 1.0, 'num_denoising_steps': 10}, 'net': {'max_horizon': 91, 'in_channels': 6, 'out_channels': 6, 'model_channels': 512, 'num_blocks': 12, 'num_heads': 8, 'mlp_ratio': 4.0, 'atten_backend': 'flash_attn_no_cp', 'crossattn_emb_channels': 2048, 'use_adaln_lora': True, 'adaln_lora_dim': 128, 'pair_timestep_feature_rank': 512, 'sac_config': {'mode': 'none', 'every_n_blocks': 1}, '_target_': <class 'cosmos_predict2.models.world2action_dit.World2ActionDIT'>}, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, '_target_': 'cosmos_predict2.configs.config_world2action.World2ActionPipelineConfig', 'xattn_layer_idx': 20}, 'video_pipe_config': {'adjust_video_noise': True, 'conditioner': {'fps': {'output_key': 'fps', 'dropout_rate': 0.0, 'dtype': None, 'input_key': 'fps', '_target_': <class 'cosmos_predict2.conditioner.ReMapkey'>}, 'padding_mask': {'output_key': 'padding_mask', 'dropout_rate': 0.0, 'dtype': None, 'input_key': 'padding_mask', '_target_': <class 'cosmos_predict2.conditioner.ReMapkey'>}, 'text': {'dropout_rate': 0.0, 'input_key': ['obs/language_embedding'], '_target_': <class 'cosmos_predict2.conditioner.TextAttr'>}, 'use_video_condition': {'output_key': 'use_video_condition', 'dropout_rate': 0.0, 'input_key': 'fps', '_target_': <class 'cosmos_predict2.conditioner.BooleanFlag'>}, '_target_': <class 'cosmos_predict2.conditioner.VideoConditioner'>}, 'conditioning_strategy': 'frame_replace', 'min_num_conditional_frames': 1, 'max_num_conditional_frames': 2, 'sigma_conditional': 0.0001, 'net': {'max_img_h': 240, 'max_img_w': 240, 'max_frames': 128, 'in_channels': 16, 'out_channels': 16, 'patch_spatial': 2, 'patch_temporal': 1, 'concat_padding_mask': True, 'model_channels': 2048, 'num_blocks': 28, 'num_heads': 16, 'atten_backend': 'minimal_a2a', 'pos_emb_cls': 'rope3d', 'pos_emb_learnable': True, 'pos_emb_interpolation': 'crop', 'use_adaln_lora': True, 'adaln_lora_dim': 256, 'rope_h_extrapolation_ratio': 3.0, 'rope_w_extrapolation_ratio': 3.0, 'rope_t_extrapolation_ratio': 1.0, 'extra_per_block_abs_pos_emb': False, 'rope_enable_fps_modulation': False, 'sac_config': {'mode': 'predict2_2b_720', 'every_n_blocks': 1, '_target_': 'cosmos_predict2.models.text2image_dit.SACConfig'}, '_target_': <class 'cosmos_predict2.models.video2world_dit.MinimalV1LVGDiT'>}, 'tokenizer': {'chunk_duration': 81, 'load_mean_std': False, 'temporal_window': 16, 'name': 'tokenizer', 'vae_pth': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth', '_target_': <class 'cosmos_predict2.tokenizers.tokenizer.TokenizerInterface'>}, 'guardrail_config': {'checkpoint_dir': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints', 'offload_model_to_cpu': True, 'enabled': False}, 'precision': 'bfloat16', 'rectified_flow_t_scaling_factor': 1.0, 'rectified_flow_loss_weight_uniform': True, 'resize_online': False, 'resolution': '480', 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0, '_target_': 'cosmos_predict2.configs.defaults.ema.EMAConfig'}, 'sigma_data': 1.0, 'state_ch': 16, 'state_t': 16, 'text_encoder': {'cls': <TextEncoderClass.T5: 't5'>, 't5': {'ckpt_path': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/text_encoder/t5-11b', 'num_tokens': 512, 'embed_dim': 1024}}, 'input_video_key': 'video', 'input_image_key': 'images', 'timestamps': {'nfe': 35, 't_min': 0.002, 't_max': 80.0, 'order': 7.0, 'is_forward': False}}, 'fsdp_shard_size': 0, 'data_config': {'config_name': 'so101_lerobot', '_target_': <function get_data_config at 0x7d90ec19dea0>}, '_target_': 'cosmos_predict2.models.world2action_model.World2ActionModelConfig'}, '_target_': <class 'cosmos_predict2.models.world2action_model.World2ActionModel'>}
+* world2action_pipe: {'precision': 'bfloat16', 'scheduler': {'alpha': 1.0, 'beta': 1.0, 'num_denoising_steps': 10}, 'net': {'max_horizon': 91, 'in_channels': 6, 'out_channels': 6, 'model_channels': 512, 'num_blocks': 12, 'num_heads': 8, 'mlp_ratio': 4.0, 'atten_backend': 'flash_attn_no_cp', 'crossattn_emb_channels': 2048, 'use_adaln_lora': True, 'adaln_lora_dim': 128, 'pair_timestep_feature_rank': 512, 'sac_config': {'mode': 'none', 'every_n_blocks': 1}, '_target_': <class 'cosmos_predict2.models.world2action_dit.World2ActionDIT'>}, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, '_target_': 'cosmos_predict2.configs.config_world2action.World2ActionPipelineConfig'}
+* optimizer: {'optim_type': 'fusedadam', 'model': None, 'lr': 9.999999999999999e-05, 'weight_decay': 0.1, 'betas': [0.9, 0.99], 'eps': 1e-08, 'master_weights': True, 'capturable': True, '_target_': <function get_base_optimizer at 0x7d90d5da1f30>}
+* scheduler: {'verbosity_interval': 0, 'warm_up_steps': [1000], 'cycle_lengths': [500000], 'f_start': [1e-06], 'f_max': [1], 'f_min': [0.2], '_target_': <class 'cosmos_predict2.functional.lr_scheduler.LambdaLinearScheduler'>}
+* data_config: {'config_name': 'so101_lerobot', '_target_': <function get_data_config at 0x7d90ec19dea0>}
+* video_dataset_train: None
+* video_dataset_val: None
+* dataloader_train: {'batch_size': {'global_bsz': 128, '_target_': <function get_local_batch_size at 0x7d90d0184dc0>}, 'shuffle': None, 'sampler': {'seed': 0, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': None, 'seed': 42, 'train': True, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, 'num_replicas': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_world_size at 0x7d934b8713f0>}, 'rank': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_rank at 0x7d934b871510>}, 'shuffle': True, '_target_': <class 'cosmos_predict2.data.resumable_sampler.ResumableDistributedSampler'>}, 'batch_sampler': None, 'num_workers': 8, 'collate_fn': None, 'pin_memory': True, 'drop_last': True, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None, 'prefetch_factor': 8, 'persistent_workers': True, 'pin_memory_device': '', 'in_order': True, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': None, 'seed': 42, 'train': True, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, '_target_': <class 'torch.utils.data.dataloader.DataLoader'>}
+* dataloader_val: {'batch_size': 1, 'shuffle': None, 'sampler': {'seed': 0, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': None, 'seed': 42, 'train': False, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, 'num_replicas': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_world_size at 0x7d934b8713f0>}, 'rank': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_rank at 0x7d934b871510>}, 'shuffle': False, '_target_': <class 'cosmos_predict2.data.resumable_sampler.ResumableDistributedSampler'>}, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None, 'prefetch_factor': None, 'persistent_workers': False, 'pin_memory_device': '', 'in_order': True, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': None, 'seed': 42, 'train': False, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, '_target_': <class 'torch.utils.data.dataloader.DataLoader'>}
+* job:
+   * project: vam
+   * group: so101
+   * name: w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5
+* trainer:
+   * type: <class 'imaginaire.trainer.ImaginaireTrainer'>
+   * callbacks: {'ema': {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.EMAModelCallback'>}, 'progress_bar': {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.ProgressBarCallback'>}, 'low_prec': {'config': None, 'trainer': None, 'update_iter': 1, '_target_': <class 'imaginaire.utils.callback.LowPrecisionCallback'>}, 'iter_speed': {'hit_thres': 5, 'every_n': 1000, '_target_': <class 'cosmos_predict2.callbacks.iter_speed.IterSpeed'>}, 'device_monitor': {'every_n': 1000, 'step_size': 1, 'log_memory_detail': True, '_target_': <class 'cosmos_predict2.callbacks.device_monitor.DeviceMonitor'>}, 'manual_gc': {'warm_up': 5, 'every_n': 5, '_target_': <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>}, 'grad_clip': {'clip_norm': 10.0, 'force_finite': True, 'log_wandb': False, '_target_': <class 'cosmos_predict2.callbacks.grad_clip.GradClip'>}, 'wandb': {'mode': None, '_target_': <class 'cosmos_predict2.callbacks.wandb_logger.WandBLogger'>}}
+   * distributed_parallelism: ddp
+   * ddp:
+      * find_unused_parameters: False
+      * static_graph: True
+      * broadcast_buffers: True
+   * cudnn:
+      * deterministic: False
+      * benchmark: True
+   * seed: 0
+   * grad_scaler_args: {'enabled': False}
+   * max_iter: 500000
+   * max_val_iter: None
+   * logging_iter: 1000
+   * run_validation: False
+   * validation_iter: 1000
+   * timeout_period: 999999999
+   * memory_format: torch.preserve_format
+   * grad_accum_iter: 1
+   * profiling:
+      * enable_profiling: False
+      * enable_memory_snapshot: False
+      * profile_freq: 1
+      * first_n_rank: 4
+      * record_shape: True
+      * profile_memory: True
+      * with_stack: True
+      * with_modules: True
+* model_parallel: ModelParallelConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=False, fp16=False, bf16=False, params_dtype=torch.float32, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.float32, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=False, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, cross_entropy_fusion_impl='native', tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=None, variable_seq_lengths=False, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=False, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=False, cpu_offloading_weights=False, barrier_with_L1_time=True)
+* checkpoint:
+   * type: {'callbacks': None, '_target_': <class 'cosmos_predict2.checkpointer.Checkpointer'>}
+   * dcp_async_mode_enabled: False
+   * save_iter: 1000
+   * load_path: 
+   * load_training_state: False
+   * only_load_scheduler_state: False
+   * strict_resume: True
+   * jit:
+      * enabled: False
+      * input_shape: None
+      * device: cuda
+      * dtype: bfloat16
+      * strict: True
+   * verbose: True
+   * keys_not_to_resume: []
+   * broadcast_via_filesystem: False
+   * load_ema_to_reg: False
+   * dcp_allow_mismatched_size: False
+* defaults: ['_self_', {'data_config': None}, {'video_dataset_train': None}, {'video_dataset_val': None}, {'dataloader_train': None}, {'dataloader_val': None}, {'world2action_pipe': None}, {'optimizer': 'fusedadamw'}, {'scheduler': 'constant'}, {'model': None}, {'callbacks': ['basic']}, {'net': None}, {'ema': None}, {'checkpoint': None}, {'ckpt_type': None}, {'experiment': None}]
+[05-15 15:55:19|WARNING|imaginaire/utils/misc.py:127:print_environ_variables] Environment variable TORCH_HOME not set!
+[05-15 15:55:19|WARNING|imaginaire/utils/misc.py:127:print_environ_variables] Environment variable IMAGINAIRE_OUTPUT_ROOT not set!
+[05-15 15:55:19|INFO|imaginaire/utils/misc.py:139:set_random_seed] Using random seed 0.
+[05-15 15:55:19|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback ema: {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.EMAModelCallback'>}
+[05-15 15:55:19|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback progress_bar: {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.ProgressBarCallback'>}
+[05-15 15:55:19|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback low_prec: {'config': None, 'trainer': None, 'update_iter': 1, '_target_': <class 'imaginaire.utils.callback.LowPrecisionCallback'>}
+[05-15 15:55:19|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback iter_speed: {'hit_thres': 5, 'every_n': 1000, '_target_': <class 'cosmos_predict2.callbacks.iter_speed.IterSpeed'>}
+[05-15 15:55:19|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback device_monitor: {'every_n': 1000, 'step_size': 1, 'log_memory_detail': True, '_target_': <class 'cosmos_predict2.callbacks.device_monitor.DeviceMonitor'>}
+[05-15 15:55:19|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback manual_gc: {'warm_up': 5, 'every_n': 5, '_target_': <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>}
+[05-15 15:55:19|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback grad_clip: {'clip_norm': 10.0, 'force_finite': True, 'log_wandb': False, '_target_': <class 'cosmos_predict2.callbacks.grad_clip.GradClip'>}
+[05-15 15:55:19|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback wandb: {'mode': None, '_target_': <class 'cosmos_predict2.callbacks.wandb_logger.WandBLogger'>}
+[05-15 15:55:21|CRITICAL|cosmos_predict2/models/world2action_model.py:106:__init__] Using mean loss reduce with loss scale 10.0
+[05-15 15:55:21|WARNING|cosmos_predict2/pipelines/world2action.py:71:from_config] precision torch.bfloat16
+[05-15 15:55:21|WARNING|cosmos_predict2/pipelines/world2action.py:101:from_config] dit_path not provided, initializing DiT with random weights
+[05-15 15:55:21|WARNING|cosmos_predict2/pipelines/video2world.py:277:from_config] precision torch.bfloat16
+[05-15 15:55:22|INFO|cosmos_predict2/tokenizers/tokenizer.py:687:_video_vae] Loading /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth
+[05-15 15:55:22|SUCCESS|cosmos_predict2/tokenizers/tokenizer.py:689:_video_vae] Successfully loaded /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth
+[05-15 15:55:22|INFO|imaginaire/utils/distributed.py:430:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0, 1].
+[05-15 15:55:22|INFO|cosmos_predict2/pipelines/video2world.py:335:from_config] Loading DiT from /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_that_thing.pt
+[05-15 16:00:27|INFO|imaginaire/trainer.py:116:__init__] Config:
+* model: {'config': {'validation_mode': 'loss_only', 'train_architecture': 'base', 'lora_rank': 16, 'lora_alpha': 16, 'lora_target_modules': 'q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2', 'init_lora_weights': True, 'precision': 'bfloat16', 'loss_reduce': 'mean', 'loss_scale': 10.0, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, 'action_dit_path': '', 'video_dit_path': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt', 'pipe_config': {'precision': 'bfloat16', 'scheduler': {'alpha': 1.0, 'beta': 1.0, 'num_denoising_steps': 10}, 'net': {'max_horizon': 91, 'in_channels': 6, 'out_channels': 6, 'model_channels': 512, 'num_blocks': 12, 'num_heads': 8, 'mlp_ratio': 4.0, 'atten_backend': 'flash_attn_no_cp', 'crossattn_emb_channels': 2048, 'use_adaln_lora': True, 'adaln_lora_dim': 128, 'pair_timestep_feature_rank': 512, 'sac_config': {'mode': 'none', 'every_n_blocks': 1}, '_target_': <class 'cosmos_predict2.models.world2action_dit.World2ActionDIT'>}, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, '_target_': 'cosmos_predict2.configs.config_world2action.World2ActionPipelineConfig', 'xattn_layer_idx': 20}, 'video_pipe_config': {'adjust_video_noise': True, 'conditioner': {'fps': {'output_key': 'fps', 'dropout_rate': 0.0, 'dtype': None, 'input_key': 'fps', '_target_': <class 'cosmos_predict2.conditioner.ReMapkey'>}, 'padding_mask': {'output_key': 'padding_mask', 'dropout_rate': 0.0, 'dtype': None, 'input_key': 'padding_mask', '_target_': <class 'cosmos_predict2.conditioner.ReMapkey'>}, 'text': {'dropout_rate': 0.0, 'input_key': ['obs/language_embedding'], '_target_': <class 'cosmos_predict2.conditioner.TextAttr'>}, 'use_video_condition': {'output_key': 'use_video_condition', 'dropout_rate': 0.0, 'input_key': 'fps', '_target_': <class 'cosmos_predict2.conditioner.BooleanFlag'>}, '_target_': <class 'cosmos_predict2.conditioner.VideoConditioner'>}, 'conditioning_strategy': 'frame_replace', 'min_num_conditional_frames': 1, 'max_num_conditional_frames': 2, 'sigma_conditional': 0.0001, 'net': {'max_img_h': 240, 'max_img_w': 240, 'max_frames': 128, 'in_channels': 16, 'out_channels': 16, 'patch_spatial': 2, 'patch_temporal': 1, 'concat_padding_mask': True, 'model_channels': 2048, 'num_blocks': 28, 'num_heads': 16, 'atten_backend': 'minimal_a2a', 'pos_emb_cls': 'rope3d', 'pos_emb_learnable': True, 'pos_emb_interpolation': 'crop', 'use_adaln_lora': True, 'adaln_lora_dim': 256, 'rope_h_extrapolation_ratio': 3.0, 'rope_w_extrapolation_ratio': 3.0, 'rope_t_extrapolation_ratio': 1.0, 'extra_per_block_abs_pos_emb': False, 'rope_enable_fps_modulation': False, 'sac_config': {'mode': 'predict2_2b_720', 'every_n_blocks': 1, '_target_': 'cosmos_predict2.models.text2image_dit.SACConfig'}, '_target_': <class 'cosmos_predict2.models.video2world_dit.MinimalV1LVGDiT'>}, 'tokenizer': {'chunk_duration': 81, 'load_mean_std': False, 'temporal_window': 16, 'name': 'tokenizer', 'vae_pth': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth', '_target_': <class 'cosmos_predict2.tokenizers.tokenizer.TokenizerInterface'>}, 'guardrail_config': {'checkpoint_dir': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints', 'offload_model_to_cpu': True, 'enabled': False}, 'precision': 'bfloat16', 'rectified_flow_t_scaling_factor': 1.0, 'rectified_flow_loss_weight_uniform': True, 'resize_online': False, 'resolution': '480', 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0, '_target_': 'cosmos_predict2.configs.defaults.ema.EMAConfig'}, 'sigma_data': 1.0, 'state_ch': 16, 'state_t': 16, 'text_encoder': {'cls': <TextEncoderClass.T5: 't5'>, 't5': {'ckpt_path': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/text_encoder/t5-11b', 'num_tokens': 512, 'embed_dim': 1024}}, 'input_video_key': 'video', 'input_image_key': 'images', 'timestamps': {'nfe': 35, 't_min': 0.002, 't_max': 80.0, 'order': 7.0, 'is_forward': False}}, 'fsdp_shard_size': 0, 'data_config': {'config_name': 'so101_lerobot', '_target_': <function get_data_config at 0x734c57fc5fc0>}, '_target_': 'cosmos_predict2.models.world2action_model.World2ActionModelConfig'}, '_target_': <class 'cosmos_predict2.models.world2action_model.World2ActionModel'>}
+* world2action_pipe: {'precision': 'bfloat16', 'scheduler': {'alpha': 1.0, 'beta': 1.0, 'num_denoising_steps': 10}, 'net': {'max_horizon': 91, 'in_channels': 6, 'out_channels': 6, 'model_channels': 512, 'num_blocks': 12, 'num_heads': 8, 'mlp_ratio': 4.0, 'atten_backend': 'flash_attn_no_cp', 'crossattn_emb_channels': 2048, 'use_adaln_lora': True, 'adaln_lora_dim': 128, 'pair_timestep_feature_rank': 512, 'sac_config': {'mode': 'none', 'every_n_blocks': 1}, '_target_': <class 'cosmos_predict2.models.world2action_dit.World2ActionDIT'>}, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, '_target_': 'cosmos_predict2.configs.config_world2action.World2ActionPipelineConfig'}
+* optimizer: {'optim_type': 'fusedadam', 'model': None, 'lr': 9.999999999999999e-05, 'weight_decay': 0.1, 'betas': [0.9, 0.99], 'eps': 1e-08, 'master_weights': True, 'capturable': True, '_target_': <function get_base_optimizer at 0x734c467b6050>}
+* scheduler: {'verbosity_interval': 0, 'warm_up_steps': [1000], 'cycle_lengths': [500000], 'f_start': [1e-06], 'f_max': [1], 'f_min': [0.2], '_target_': <class 'cosmos_predict2.functional.lr_scheduler.LambdaLinearScheduler'>}
+* data_config: {'config_name': 'so101_lerobot', '_target_': <function get_data_config at 0x734c57fc5fc0>}
+* video_dataset_train: None
+* video_dataset_val: None
+* dataloader_train: {'batch_size': 2, 'shuffle': None, 'sampler': {'seed': 0, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': None, 'seed': 42, 'train': True, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, 'num_replicas': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_world_size at 0x734ebb46d510>}, 'rank': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_rank at 0x734ebb46d630>}, 'shuffle': True, '_target_': <class 'cosmos_predict2.data.resumable_sampler.ResumableDistributedSampler'>}, 'batch_sampler': None, 'num_workers': 8, 'collate_fn': None, 'pin_memory': True, 'drop_last': True, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None, 'prefetch_factor': 8, 'persistent_workers': True, 'pin_memory_device': '', 'in_order': True, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': 0.05, 'seed': 42, 'train': True, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, '_target_': <class 'torch.utils.data.dataloader.DataLoader'>}
+* dataloader_val: {'batch_size': 1, 'shuffle': None, 'sampler': {'seed': 0, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': None, 'seed': 42, 'train': False, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, 'num_replicas': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_world_size at 0x734ebb46d510>}, 'rank': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_rank at 0x734ebb46d630>}, 'shuffle': False, '_target_': <class 'cosmos_predict2.data.resumable_sampler.ResumableDistributedSampler'>}, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None, 'prefetch_factor': None, 'persistent_workers': False, 'pin_memory_device': '', 'in_order': True, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': 0.05, 'seed': 42, 'train': False, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, '_target_': <class 'torch.utils.data.dataloader.DataLoader'>}
+* job:
+   * project: vam
+   * group: so101
+   * name: w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5
+* trainer:
+   * type: <class 'imaginaire.trainer.ImaginaireTrainer'>
+   * callbacks: {'ema': {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.EMAModelCallback'>}, 'progress_bar': {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.ProgressBarCallback'>}, 'low_prec': {'config': None, 'trainer': None, 'update_iter': 1, '_target_': <class 'imaginaire.utils.callback.LowPrecisionCallback'>}, 'iter_speed': {'hit_thres': 5, 'every_n': 1000, '_target_': <class 'cosmos_predict2.callbacks.iter_speed.IterSpeed'>}, 'device_monitor': {'every_n': 1000, 'step_size': 1, 'log_memory_detail': True, '_target_': <class 'cosmos_predict2.callbacks.device_monitor.DeviceMonitor'>}, 'manual_gc': {'warm_up': 5, 'every_n': 5, '_target_': <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>}, 'grad_clip': {'clip_norm': 10.0, 'force_finite': True, 'log_wandb': False, '_target_': <class 'cosmos_predict2.callbacks.grad_clip.GradClip'>}, 'wandb': {'mode': None, '_target_': <class 'cosmos_predict2.callbacks.wandb_logger.WandBLogger'>}}
+   * distributed_parallelism: ddp
+   * ddp:
+      * find_unused_parameters: False
+      * static_graph: True
+      * broadcast_buffers: True
+   * cudnn:
+      * deterministic: False
+      * benchmark: True
+   * seed: 0
+   * grad_scaler_args: {'enabled': False}
+   * max_iter: 8000
+   * max_val_iter: 16
+   * logging_iter: 1000
+   * run_validation: True
+   * validation_iter: 100
+   * timeout_period: 999999999
+   * memory_format: torch.preserve_format
+   * grad_accum_iter: 8
+   * profiling:
+      * enable_profiling: False
+      * enable_memory_snapshot: False
+      * profile_freq: 1
+      * first_n_rank: 4
+      * record_shape: True
+      * profile_memory: True
+      * with_stack: True
+      * with_modules: True
+* model_parallel: ModelParallelConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=False, fp16=False, bf16=False, params_dtype=torch.float32, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.float32, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=False, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, cross_entropy_fusion_impl='native', tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=None, variable_seq_lengths=False, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=False, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=False, cpu_offloading_weights=False, barrier_with_L1_time=True)
+* checkpoint:
+   * type: {'callbacks': None, '_target_': <class 'cosmos_predict2.checkpointer.Checkpointer'>}
+   * dcp_async_mode_enabled: False
+   * save_iter: 100
+   * load_path: 
+   * load_training_state: False
+   * only_load_scheduler_state: False
+   * strict_resume: True
+   * jit:
+      * enabled: False
+      * input_shape: None
+      * device: cuda
+      * dtype: bfloat16
+      * strict: True
+   * verbose: True
+   * keys_not_to_resume: []
+   * broadcast_via_filesystem: False
+   * load_ema_to_reg: False
+   * dcp_allow_mismatched_size: False
+* defaults: ['_self_', {'data_config': None}, {'video_dataset_train': None}, {'video_dataset_val': None}, {'dataloader_train': None}, {'dataloader_val': None}, {'world2action_pipe': None}, {'optimizer': 'fusedadamw'}, {'scheduler': 'constant'}, {'model': None}, {'callbacks': ['basic']}, {'net': None}, {'ema': None}, {'checkpoint': None}, {'ckpt_type': None}, {'experiment': None}]
+[05-15 16:00:27|WARNING|imaginaire/utils/misc.py:127:print_environ_variables] Environment variable TORCH_HOME not set!
+[05-15 16:00:27|WARNING|imaginaire/utils/misc.py:127:print_environ_variables] Environment variable IMAGINAIRE_OUTPUT_ROOT not set!
+[05-15 16:00:27|INFO|imaginaire/utils/misc.py:139:set_random_seed] Using random seed 0.
+[05-15 16:00:27|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback ema: {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.EMAModelCallback'>}
+[05-15 16:00:27|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback progress_bar: {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.ProgressBarCallback'>}
+[05-15 16:00:27|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback low_prec: {'config': None, 'trainer': None, 'update_iter': 1, '_target_': <class 'imaginaire.utils.callback.LowPrecisionCallback'>}
+[05-15 16:00:27|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback iter_speed: {'hit_thres': 5, 'every_n': 1000, '_target_': <class 'cosmos_predict2.callbacks.iter_speed.IterSpeed'>}
+[05-15 16:00:27|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback device_monitor: {'every_n': 1000, 'step_size': 1, 'log_memory_detail': True, '_target_': <class 'cosmos_predict2.callbacks.device_monitor.DeviceMonitor'>}
+[05-15 16:00:27|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback manual_gc: {'warm_up': 5, 'every_n': 5, '_target_': <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>}
+[05-15 16:00:27|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback grad_clip: {'clip_norm': 10.0, 'force_finite': True, 'log_wandb': False, '_target_': <class 'cosmos_predict2.callbacks.grad_clip.GradClip'>}
+[05-15 16:00:27|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback wandb: {'mode': None, '_target_': <class 'cosmos_predict2.callbacks.wandb_logger.WandBLogger'>}
+[05-15 16:00:29|CRITICAL|cosmos_predict2/models/world2action_model.py:106:__init__] Using mean loss reduce with loss scale 10.0
+[05-15 16:00:29|WARNING|cosmos_predict2/pipelines/world2action.py:71:from_config] precision torch.bfloat16
+[05-15 16:00:29|WARNING|cosmos_predict2/pipelines/world2action.py:101:from_config] dit_path not provided, initializing DiT with random weights
+[05-15 16:00:29|WARNING|cosmos_predict2/pipelines/video2world.py:277:from_config] precision torch.bfloat16
+[05-15 16:00:30|INFO|cosmos_predict2/tokenizers/tokenizer.py:687:_video_vae] Loading /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth
+[05-15 16:00:30|SUCCESS|cosmos_predict2/tokenizers/tokenizer.py:689:_video_vae] Successfully loaded /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth
+[05-15 16:00:30|INFO|imaginaire/utils/distributed.py:430:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0, 1].
+[05-15 16:00:30|INFO|cosmos_predict2/pipelines/video2world.py:335:from_config] Loading DiT from /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt
+[05-15 16:00:34|SUCCESS|cosmos_predict2/pipelines/video2world.py:354:from_config] Successfully loaded DiT from /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt
+[05-15 16:00:35|INFO|cosmos_predict2/models/world2action_model.py:148:__init__] Total parameters: 2.04B, Frozen parameters: 1,956,413,440, Trainable parameters: 80,859,000
+[05-15 16:00:35|INFO|cosmos_predict2/models/world2action_model.py:167:__init__] FSDP (Fully Sharded Data Parallel) is disabled.
+[05-15 16:00:35|CRITICAL|cosmos_predict2/utils/optim_instantiate_dtensor.py:49:get_base_optimizer] total num parameters : 80,859,000
+[05-15 16:00:35|WARNING|cosmos_predict2/utils/fused_adam_dtensor.py:103:__init__] FusedAdam master_weights: True capturable: True
+[05-15 16:00:35|INFO|cosmos_predict2/checkpointer.py:288:load] Training from scratch.
+[05-15 16:00:35|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint loading: 0.0002 seconds
+[05-15 16:00:35|CRITICAL|imaginaire/trainer.py:179:train] Distributed parallelism mode: ddp
+[05-15 16:00:36|INFO|imaginaire/trainer.py:187:train] Starting training...
+[05-15 16:00:36|INFO|cosmos_predict2/callbacks/device_monitor.py:92:on_train_start] DeviceMonitor callback: local_dir: checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/DeviceMonitor
+[05-15 16:00:37|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 0...
+[05-15 16:01:28|INFO|imaginaire/trainer.py:193:train] Initial validation done.
+[05-15 16:01:31|WARNING|imaginaire/utils/distributed.py:284:ddp_sync_grad] DDP static_graph=True is incompatible with sync_grad(). Performance will be reduced.
+[05-15 16:01:51|INFO|imaginaire/trainer.py:116:__init__] Config:
+* model: {'config': {'validation_mode': 'loss_only', 'train_architecture': 'base', 'lora_rank': 16, 'lora_alpha': 16, 'lora_target_modules': 'q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2', 'init_lora_weights': True, 'precision': 'bfloat16', 'loss_reduce': 'mean', 'loss_scale': 10.0, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, 'action_dit_path': '', 'video_dit_path': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt', 'pipe_config': {'precision': 'bfloat16', 'scheduler': {'alpha': 1.0, 'beta': 1.0, 'num_denoising_steps': 10}, 'net': {'max_horizon': 91, 'in_channels': 6, 'out_channels': 6, 'model_channels': 512, 'num_blocks': 12, 'num_heads': 8, 'mlp_ratio': 4.0, 'atten_backend': 'flash_attn_no_cp', 'crossattn_emb_channels': 2048, 'use_adaln_lora': True, 'adaln_lora_dim': 128, 'pair_timestep_feature_rank': 512, 'sac_config': {'mode': 'none', 'every_n_blocks': 1}, '_target_': <class 'cosmos_predict2.models.world2action_dit.World2ActionDIT'>}, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, '_target_': 'cosmos_predict2.configs.config_world2action.World2ActionPipelineConfig', 'xattn_layer_idx': 20}, 'video_pipe_config': {'adjust_video_noise': True, 'conditioner': {'fps': {'output_key': 'fps', 'dropout_rate': 0.0, 'dtype': None, 'input_key': 'fps', '_target_': <class 'cosmos_predict2.conditioner.ReMapkey'>}, 'padding_mask': {'output_key': 'padding_mask', 'dropout_rate': 0.0, 'dtype': None, 'input_key': 'padding_mask', '_target_': <class 'cosmos_predict2.conditioner.ReMapkey'>}, 'text': {'dropout_rate': 0.0, 'input_key': ['obs/language_embedding'], '_target_': <class 'cosmos_predict2.conditioner.TextAttr'>}, 'use_video_condition': {'output_key': 'use_video_condition', 'dropout_rate': 0.0, 'input_key': 'fps', '_target_': <class 'cosmos_predict2.conditioner.BooleanFlag'>}, '_target_': <class 'cosmos_predict2.conditioner.VideoConditioner'>}, 'conditioning_strategy': 'frame_replace', 'min_num_conditional_frames': 1, 'max_num_conditional_frames': 2, 'sigma_conditional': 0.0001, 'net': {'max_img_h': 240, 'max_img_w': 240, 'max_frames': 128, 'in_channels': 16, 'out_channels': 16, 'patch_spatial': 2, 'patch_temporal': 1, 'concat_padding_mask': True, 'model_channels': 2048, 'num_blocks': 28, 'num_heads': 16, 'atten_backend': 'minimal_a2a', 'pos_emb_cls': 'rope3d', 'pos_emb_learnable': True, 'pos_emb_interpolation': 'crop', 'use_adaln_lora': True, 'adaln_lora_dim': 256, 'rope_h_extrapolation_ratio': 3.0, 'rope_w_extrapolation_ratio': 3.0, 'rope_t_extrapolation_ratio': 1.0, 'extra_per_block_abs_pos_emb': False, 'rope_enable_fps_modulation': False, 'sac_config': {'mode': 'predict2_2b_720', 'every_n_blocks': 1, '_target_': 'cosmos_predict2.models.text2image_dit.SACConfig'}, '_target_': <class 'cosmos_predict2.models.video2world_dit.MinimalV1LVGDiT'>}, 'tokenizer': {'chunk_duration': 81, 'load_mean_std': False, 'temporal_window': 16, 'name': 'tokenizer', 'vae_pth': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth', '_target_': <class 'cosmos_predict2.tokenizers.tokenizer.TokenizerInterface'>}, 'guardrail_config': {'checkpoint_dir': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints', 'offload_model_to_cpu': True, 'enabled': False}, 'precision': 'bfloat16', 'rectified_flow_t_scaling_factor': 1.0, 'rectified_flow_loss_weight_uniform': True, 'resize_online': False, 'resolution': '480', 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0, '_target_': 'cosmos_predict2.configs.defaults.ema.EMAConfig'}, 'sigma_data': 1.0, 'state_ch': 16, 'state_t': 16, 'text_encoder': {'cls': <TextEncoderClass.T5: 't5'>, 't5': {'ckpt_path': '/home/nvidia/pdt-mimic/mimic-video/model/checkpoints/text_encoder/t5-11b', 'num_tokens': 512, 'embed_dim': 1024}}, 'input_video_key': 'video', 'input_image_key': 'images', 'timestamps': {'nfe': 35, 't_min': 0.002, 't_max': 80.0, 'order': 7.0, 'is_forward': False}}, 'fsdp_shard_size': 0, 'data_config': {'config_name': 'so101_lerobot', '_target_': <function get_data_config at 0x70f0db8c5fc0>}, '_target_': 'cosmos_predict2.models.world2action_model.World2ActionModelConfig'}, '_target_': <class 'cosmos_predict2.models.world2action_model.World2ActionModel'>}
+* world2action_pipe: {'precision': 'bfloat16', 'scheduler': {'alpha': 1.0, 'beta': 1.0, 'num_denoising_steps': 10}, 'net': {'max_horizon': 91, 'in_channels': 6, 'out_channels': 6, 'model_channels': 512, 'num_blocks': 12, 'num_heads': 8, 'mlp_ratio': 4.0, 'atten_backend': 'flash_attn_no_cp', 'crossattn_emb_channels': 2048, 'use_adaln_lora': True, 'adaln_lora_dim': 128, 'pair_timestep_feature_rank': 512, 'sac_config': {'mode': 'none', 'every_n_blocks': 1}, '_target_': <class 'cosmos_predict2.models.world2action_dit.World2ActionDIT'>}, 'ema': {'enabled': False, 'rate': 0.1, 'iteration_shift': 0}, '_target_': 'cosmos_predict2.configs.config_world2action.World2ActionPipelineConfig'}
+* optimizer: {'optim_type': 'fusedadam', 'model': None, 'lr': 9.999999999999999e-05, 'weight_decay': 0.1, 'betas': [0.9, 0.99], 'eps': 1e-08, 'master_weights': True, 'capturable': True, '_target_': <function get_base_optimizer at 0x70f0c93b6050>}
+* scheduler: {'verbosity_interval': 0, 'warm_up_steps': [1000], 'cycle_lengths': [500000], 'f_start': [1e-06], 'f_max': [1], 'f_min': [0.2], '_target_': <class 'cosmos_predict2.functional.lr_scheduler.LambdaLinearScheduler'>}
+* data_config: {'config_name': 'so101_lerobot', '_target_': <function get_data_config at 0x70f0db8c5fc0>}
+* video_dataset_train: None
+* video_dataset_val: None
+* dataloader_train: {'batch_size': 2, 'shuffle': None, 'sampler': {'seed': 0, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': None, 'seed': 42, 'train': True, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, 'num_replicas': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_world_size at 0x70f33ef7d510>}, 'rank': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_rank at 0x70f33ef7d630>}, 'shuffle': True, '_target_': <class 'cosmos_predict2.data.resumable_sampler.ResumableDistributedSampler'>}, 'batch_sampler': None, 'num_workers': 8, 'collate_fn': None, 'pin_memory': True, 'drop_last': True, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None, 'prefetch_factor': 8, 'persistent_workers': True, 'pin_memory_device': '', 'in_order': True, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': 0.05, 'seed': 42, 'train': True, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, '_target_': <class 'torch.utils.data.dataloader.DataLoader'>}
+* dataloader_val: {'batch_size': 1, 'shuffle': None, 'sampler': {'seed': 0, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': None, 'seed': 42, 'train': False, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, 'num_replicas': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_world_size at 0x70f33ef7d510>}, 'rank': {'with_context_parallel': False, 'partial_data_parallel': False, '_target_': <function get_data_parallel_rank at 0x70f33ef7d630>}, 'shuffle': False, '_target_': <class 'cosmos_predict2.data.resumable_sampler.ResumableDistributedSampler'>}, 'batch_sampler': None, 'num_workers': 0, 'collate_fn': None, 'pin_memory': False, 'drop_last': False, 'timeout': 0, 'worker_init_fn': None, 'multiprocessing_context': None, 'generator': None, 'prefetch_factor': None, 'persistent_workers': False, 'pin_memory_device': '', 'in_order': True, 'dataset': {'root': None, 'image_key': 'observation.images.front', 'state_key': 'observation.state', 'action_key': 'action', 'obs_image_horizon': 5, 'action_image_horizon': 56, 'action_lowdim_horizon': 90, 'target_fps': 5.0, 'lowdim_target_fps': 30.0, 'action_shift_s': 0.2, 'image_resize': [480, 640], 'language_embeddings_path': None, 't5_encoder_ckpt': None, 'num_val_episodes': 1, 'val_ratio': 0.05, 'seed': 42, 'train': False, 'cache_dir': None, 'video_backend': 'pyav', 'repo_id': 'push-that-thing/task_1', '_target_': <class 'cosmos_predict2.data.action.dataset_lerobot.LeRobotDataset'>}, '_target_': <class 'torch.utils.data.dataloader.DataLoader'>}
+* job:
+   * project: vam
+   * group: so101
+   * name: w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5
+* trainer:
+   * type: <class 'imaginaire.trainer.ImaginaireTrainer'>
+   * callbacks: {'ema': {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.EMAModelCallback'>}, 'progress_bar': {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.ProgressBarCallback'>}, 'low_prec': {'config': None, 'trainer': None, 'update_iter': 1, '_target_': <class 'imaginaire.utils.callback.LowPrecisionCallback'>}, 'iter_speed': {'hit_thres': 5, 'every_n': 1000, '_target_': <class 'cosmos_predict2.callbacks.iter_speed.IterSpeed'>}, 'device_monitor': {'every_n': 1000, 'step_size': 1, 'log_memory_detail': True, '_target_': <class 'cosmos_predict2.callbacks.device_monitor.DeviceMonitor'>}, 'manual_gc': {'warm_up': 5, 'every_n': 5, '_target_': <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>}, 'grad_clip': {'clip_norm': 10.0, 'force_finite': True, 'log_wandb': False, '_target_': <class 'cosmos_predict2.callbacks.grad_clip.GradClip'>}, 'wandb': {'mode': None, '_target_': <class 'cosmos_predict2.callbacks.wandb_logger.WandBLogger'>}}
+   * distributed_parallelism: ddp
+   * ddp:
+      * find_unused_parameters: False
+      * static_graph: True
+      * broadcast_buffers: True
+   * cudnn:
+      * deterministic: False
+      * benchmark: True
+   * seed: 0
+   * grad_scaler_args: {'enabled': False}
+   * max_iter: 8000
+   * max_val_iter: 16
+   * logging_iter: 1000
+   * run_validation: True
+   * validation_iter: 100
+   * timeout_period: 999999999
+   * memory_format: torch.preserve_format
+   * grad_accum_iter: 8
+   * profiling:
+      * enable_profiling: False
+      * enable_memory_snapshot: False
+      * profile_freq: 1
+      * first_n_rank: 4
+      * record_shape: True
+      * profile_memory: True
+      * with_stack: True
+      * with_modules: True
+* model_parallel: ModelParallelConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=False, fp16=False, bf16=False, params_dtype=torch.float32, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.float32, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=False, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, cross_entropy_fusion_impl='native', tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=None, variable_seq_lengths=False, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=False, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=False, cpu_offloading_weights=False, barrier_with_L1_time=True)
+* checkpoint:
+   * type: {'callbacks': None, '_target_': <class 'cosmos_predict2.checkpointer.Checkpointer'>}
+   * dcp_async_mode_enabled: False
+   * save_iter: 100
+   * load_path: 
+   * load_training_state: False
+   * only_load_scheduler_state: False
+   * strict_resume: True
+   * jit:
+      * enabled: False
+      * input_shape: None
+      * device: cuda
+      * dtype: bfloat16
+      * strict: True
+   * verbose: True
+   * keys_not_to_resume: []
+   * broadcast_via_filesystem: False
+   * load_ema_to_reg: False
+   * dcp_allow_mismatched_size: False
+* defaults: ['_self_', {'data_config': None}, {'video_dataset_train': None}, {'video_dataset_val': None}, {'dataloader_train': None}, {'dataloader_val': None}, {'world2action_pipe': None}, {'optimizer': 'fusedadamw'}, {'scheduler': 'constant'}, {'model': None}, {'callbacks': ['basic']}, {'net': None}, {'ema': None}, {'checkpoint': None}, {'ckpt_type': None}, {'experiment': None}]
+[05-15 16:01:51|WARNING|imaginaire/utils/misc.py:127:print_environ_variables] Environment variable TORCH_HOME not set!
+[05-15 16:01:51|WARNING|imaginaire/utils/misc.py:127:print_environ_variables] Environment variable IMAGINAIRE_OUTPUT_ROOT not set!
+[05-15 16:01:51|INFO|imaginaire/utils/misc.py:139:set_random_seed] Using random seed 0.
+[05-15 16:01:51|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback ema: {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.EMAModelCallback'>}
+[05-15 16:01:51|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback progress_bar: {'config': None, 'trainer': None, '_target_': <class 'imaginaire.utils.callback.ProgressBarCallback'>}
+[05-15 16:01:51|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback low_prec: {'config': None, 'trainer': None, 'update_iter': 1, '_target_': <class 'imaginaire.utils.callback.LowPrecisionCallback'>}
+[05-15 16:01:51|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback iter_speed: {'hit_thres': 5, 'every_n': 1000, '_target_': <class 'cosmos_predict2.callbacks.iter_speed.IterSpeed'>}
+[05-15 16:01:51|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback device_monitor: {'every_n': 1000, 'step_size': 1, 'log_memory_detail': True, '_target_': <class 'cosmos_predict2.callbacks.device_monitor.DeviceMonitor'>}
+[05-15 16:01:51|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback manual_gc: {'warm_up': 5, 'every_n': 5, '_target_': <class 'imaginaire.callbacks.manual_gc.ManualGarbageCollection'>}
+[05-15 16:01:51|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback grad_clip: {'clip_norm': 10.0, 'force_finite': True, 'log_wandb': False, '_target_': <class 'cosmos_predict2.callbacks.grad_clip.GradClip'>}
+[05-15 16:01:51|CRITICAL|imaginaire/utils/callback.py:77:__init__] Instantiating callback wandb: {'mode': None, '_target_': <class 'cosmos_predict2.callbacks.wandb_logger.WandBLogger'>}
+[05-15 16:01:54|CRITICAL|cosmos_predict2/models/world2action_model.py:106:__init__] Using mean loss reduce with loss scale 10.0
+[05-15 16:01:54|WARNING|cosmos_predict2/pipelines/world2action.py:71:from_config] precision torch.bfloat16
+[05-15 16:01:54|WARNING|cosmos_predict2/pipelines/world2action.py:101:from_config] dit_path not provided, initializing DiT with random weights
+[05-15 16:01:54|WARNING|cosmos_predict2/pipelines/video2world.py:277:from_config] precision torch.bfloat16
+[05-15 16:01:55|INFO|cosmos_predict2/tokenizers/tokenizer.py:687:_video_vae] Loading /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth
+[05-15 16:01:55|SUCCESS|cosmos_predict2/tokenizers/tokenizer.py:689:_video_vae] Successfully loaded /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone//tokenizer/tokenizer.pth
+[05-15 16:01:55|INFO|imaginaire/utils/distributed.py:430:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0, 1].
+[05-15 16:01:55|INFO|cosmos_predict2/pipelines/video2world.py:335:from_config] Loading DiT from /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt
+[05-15 16:02:01|SUCCESS|cosmos_predict2/pipelines/video2world.py:354:from_config] Successfully loaded DiT from /home/nvidia/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt
+[05-15 16:02:02|INFO|cosmos_predict2/models/world2action_model.py:148:__init__] Total parameters: 2.04B, Frozen parameters: 1,956,413,440, Trainable parameters: 80,859,000
+[05-15 16:02:02|INFO|cosmos_predict2/models/world2action_model.py:167:__init__] FSDP (Fully Sharded Data Parallel) is disabled.
+[05-15 16:02:03|CRITICAL|cosmos_predict2/utils/optim_instantiate_dtensor.py:49:get_base_optimizer] total num parameters : 80,859,000
+[05-15 16:02:03|WARNING|cosmos_predict2/utils/fused_adam_dtensor.py:103:__init__] FusedAdam master_weights: True capturable: True
+[05-15 16:02:03|INFO|cosmos_predict2/checkpointer.py:288:load] Training from scratch.
+[05-15 16:02:03|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint loading: 0.0002 seconds
+[05-15 16:02:03|CRITICAL|imaginaire/trainer.py:179:train] Distributed parallelism mode: ddp
+[05-15 16:02:04|INFO|imaginaire/trainer.py:187:train] Starting training...
+[05-15 16:02:04|INFO|cosmos_predict2/callbacks/device_monitor.py:92:on_train_start] DeviceMonitor callback: local_dir: checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/DeviceMonitor
+[05-15 16:02:05|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 0...
+[05-15 16:02:55|INFO|imaginaire/trainer.py:193:train] Initial validation done.
+[05-15 16:02:59|WARNING|imaginaire/utils/distributed.py:284:ddp_sync_grad] DDP static_graph=True is incompatible with sync_grad(). Performance will be reduced.
+[05-15 16:03:30|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 1: Hit counter: 1/5 | Loss: 29.3435 | Time: 98.82s
+[05-15 16:03:48|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 2: Hit counter: 2/5 | Loss: 40.5877 | Time: 17.63s
+[05-15 16:04:05|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 3: Hit counter: 3/5 | Loss: 37.0929 | Time: 17.54s
+[05-15 16:04:23|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 4: Hit counter: 4/5 | Loss: 27.8658 | Time: 17.49s
+[05-15 16:04:40|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 5: Hit counter: 5/5 | Loss: 23.2484 | Time: 17.56s
+[05-15 16:10:29|CRITICAL|imaginaire/callbacks/manual_gc.py:48:every_n_impl] Garbage collection disabled
+[05-15 16:32:17|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000100.pt
+[05-15 16:32:17|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1471 seconds
+[05-15 16:32:18|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000100.pt
+[05-15 16:32:18|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5619 seconds
+[05-15 16:32:18|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000100.pt
+[05-15 16:32:18|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0018 seconds
+[05-15 16:32:18|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000100.pt
+[05-15 16:32:18|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
+[05-15 16:32:18|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 100...
+[05-15 17:02:07|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000200.pt
+[05-15 17:02:07|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1504 seconds
+[05-15 17:02:08|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000200.pt
+[05-15 17:02:08|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5702 seconds
+[05-15 17:02:08|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000200.pt
+[05-15 17:02:08|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0016 seconds
+[05-15 17:02:08|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000200.pt
+[05-15 17:02:08|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0013 seconds
+[05-15 17:02:08|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 200...
+[05-15 17:32:00|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000300.pt
+[05-15 17:32:00|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1468 seconds
+[05-15 17:32:01|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000300.pt
+[05-15 17:32:01|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6696 seconds
+[05-15 17:32:01|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000300.pt
+[05-15 17:32:01|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0028 seconds
+[05-15 17:32:01|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000300.pt
+[05-15 17:32:01|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0012 seconds
+[05-15 17:32:01|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 300...
+[05-15 18:01:55|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000400.pt
+[05-15 18:01:55|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1452 seconds
+[05-15 18:01:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000400.pt
+[05-15 18:01:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6741 seconds
+[05-15 18:01:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000400.pt
+[05-15 18:01:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0018 seconds
+[05-15 18:01:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000400.pt
+[05-15 18:01:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0036 seconds
+[05-15 18:01:56|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 400...
+[05-15 18:31:48|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000500.pt
+[05-15 18:31:48|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1467 seconds
+[05-15 18:31:49|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000500.pt
+[05-15 18:31:49|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5811 seconds
+[05-15 18:31:49|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000500.pt
+[05-15 18:31:49|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0020 seconds
+[05-15 18:31:49|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000500.pt
+[05-15 18:31:49|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0024 seconds
+[05-15 18:31:49|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 500...
+[05-15 19:01:38|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000600.pt
+[05-15 19:01:38|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1733 seconds
+[05-15 19:01:39|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000600.pt
+[05-15 19:01:39|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6766 seconds
+[05-15 19:01:39|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000600.pt
+[05-15 19:01:39|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0018 seconds
+[05-15 19:01:39|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000600.pt
+[05-15 19:01:39|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0016 seconds
+[05-15 19:01:39|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 600...
+[05-15 19:31:31|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000700.pt
+[05-15 19:31:31|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1505 seconds
+[05-15 19:31:32|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000700.pt
+[05-15 19:31:32|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6392 seconds
+[05-15 19:31:32|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000700.pt
+[05-15 19:31:32|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0016 seconds
+[05-15 19:31:32|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000700.pt
+[05-15 19:31:32|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
+[05-15 19:31:32|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 700...
+[05-15 20:01:25|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000800.pt
+[05-15 20:01:25|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1821 seconds
+[05-15 20:01:25|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000800.pt
+[05-15 20:01:25|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5866 seconds
+[05-15 20:01:25|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000800.pt
+[05-15 20:01:25|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0011 seconds
+[05-15 20:01:25|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000800.pt
+[05-15 20:01:25|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0023 seconds
+[05-15 20:01:25|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 800...
+[05-15 20:31:18|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000000900.pt
+[05-15 20:31:18|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1630 seconds
+[05-15 20:31:19|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000000900.pt
+[05-15 20:31:19|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5997 seconds
+[05-15 20:31:19|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000000900.pt
+[05-15 20:31:19|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0010 seconds
+[05-15 20:31:19|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000900.pt
+[05-15 20:31:19|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
+[05-15 20:31:19|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
+[05-15 21:01:12|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    67.027685      67.164883      66.890488
+peak_gpu_mem_gb               18.605011      18.611500      18.598523
+peak_gpu_mem_reserved_gb      28.186523      28.308594      28.064453
+nvml_used_gpu_mem_gb          30.916321      31.066711      30.765930
+nvml_free_gpu_mem_gb          62.667664      62.818054      62.517273
+temp                          72.500000      73.000000      72.000000
+power                     378464.500000  382747.000000  374182.000000
+util                          73.000000     100.000000      46.000000
+clock                       1740.000000    1770.000000    1710.000000
+[05-15 21:01:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001000.pt
+[05-15 21:01:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1449 seconds
+[05-15 21:01:14|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001000.pt
+[05-15 21:01:14|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6881 seconds
+[05-15 21:01:14|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001000.pt
+[05-15 21:01:14|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0019 seconds
+[05-15 21:01:14|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001000.pt
+[05-15 21:01:14|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0034 seconds
+[05-15 21:01:14|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1000...
+[05-15 21:31:07|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001100.pt
+[05-15 21:31:07|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1495 seconds
+[05-15 21:31:08|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001100.pt
+[05-15 21:31:08|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6242 seconds
+[05-15 21:31:08|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001100.pt
+[05-15 21:31:08|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0019 seconds
+[05-15 21:31:08|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001100.pt
+[05-15 21:31:08|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0029 seconds
+[05-15 21:31:08|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1100...
+[05-15 22:00:59|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001200.pt
+[05-15 22:00:59|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1707 seconds
+[05-15 22:01:00|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001200.pt
+[05-15 22:01:00|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.7220 seconds
+[05-15 22:01:00|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001200.pt
+[05-15 22:01:00|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0018 seconds
+[05-15 22:01:00|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001200.pt
+[05-15 22:01:00|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0027 seconds
+[05-15 22:01:00|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1200...
+[05-15 22:30:54|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001300.pt
+[05-15 22:30:54|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1554 seconds
+[05-15 22:30:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001300.pt
+[05-15 22:30:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.7021 seconds
+[05-15 22:30:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001300.pt
+[05-15 22:30:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0023 seconds
+[05-15 22:30:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001300.pt
+[05-15 22:30:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
+[05-15 22:30:56|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1300...
+[05-15 23:00:49|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001400.pt
+[05-15 23:00:49|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1491 seconds
+[05-15 23:00:50|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001400.pt
+[05-15 23:00:50|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6037 seconds
+[05-15 23:00:50|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001400.pt
+[05-15 23:00:50|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0020 seconds
+[05-15 23:00:50|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001400.pt
+[05-15 23:00:50|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0018 seconds
+[05-15 23:00:51|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1400...
+[05-15 23:30:44|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001500.pt
+[05-15 23:30:44|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1701 seconds
+[05-15 23:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001500.pt
+[05-15 23:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.7086 seconds
+[05-15 23:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001500.pt
+[05-15 23:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0013 seconds
+[05-15 23:30:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001500.pt
+[05-15 23:30:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0026 seconds
+[05-15 23:30:46|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1500...
+[05-16 00:00:38|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001600.pt
+[05-16 00:00:38|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1458 seconds
+[05-16 00:00:39|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001600.pt
+[05-16 00:00:39|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5767 seconds
+[05-16 00:00:39|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001600.pt
+[05-16 00:00:39|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0016 seconds
+[05-16 00:00:39|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001600.pt
+[05-16 00:00:39|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
+[05-16 00:00:39|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1600...
+[05-16 00:30:31|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001700.pt
+[05-16 00:30:31|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1660 seconds
+[05-16 00:30:32|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001700.pt
+[05-16 00:30:32|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.7120 seconds
+[05-16 00:30:32|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001700.pt
+[05-16 00:30:32|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0021 seconds
+[05-16 00:30:32|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001700.pt
+[05-16 00:30:32|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0030 seconds
+[05-16 00:30:32|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1700...
+[05-16 01:00:27|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001800.pt
+[05-16 01:00:27|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1521 seconds
+[05-16 01:00:28|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001800.pt
+[05-16 01:00:28|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5770 seconds
+[05-16 01:00:28|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001800.pt
+[05-16 01:00:28|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0021 seconds
+[05-16 01:00:28|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001800.pt
+[05-16 01:00:28|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0013 seconds
+[05-16 01:00:28|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1800...
+[05-16 01:30:19|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001900.pt
+[05-16 01:30:19|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1470 seconds
+[05-16 01:30:20|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001900.pt
+[05-16 01:30:20|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6324 seconds
+[05-16 01:30:20|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001900.pt
+[05-16 01:30:20|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0018 seconds
+[05-16 01:30:20|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001900.pt
+[05-16 01:30:20|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0021 seconds
+[05-16 01:30:20|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1900...
+[05-16 02:00:13|INFO|cosmos_predict2/callbacks/iter_speed.py:80:every_n_impl] 2000 : iter_speed 17.94 seconds per iteration | Loss: 3.5360
+[05-16 02:00:13|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    67.262220      67.424496      67.099945
+peak_gpu_mem_gb               18.605011      18.611500      18.598523
+peak_gpu_mem_reserved_gb      28.186523      28.308594      28.064453
+nvml_used_gpu_mem_gb          30.754211      30.765930      30.742493
+nvml_free_gpu_mem_gb          62.829773      62.841492      62.818054
+temp                          71.000000      72.000000      70.000000
+power                     369968.000000  376959.000000  362977.000000
+util                          89.500000     100.000000      79.000000
+clock                       1755.000000    1770.000000    1740.000000
+[05-16 02:00:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002000.pt
+[05-16 02:00:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1682 seconds
+[05-16 02:00:14|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002000.pt
+[05-16 02:00:14|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6610 seconds
+[05-16 02:00:14|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002000.pt
+[05-16 02:00:14|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0022 seconds
+[05-16 02:00:14|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002000.pt
+[05-16 02:00:14|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0019 seconds
+[05-16 02:00:15|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2000...
+[05-16 02:30:04|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002100.pt
+[05-16 02:30:04|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1475 seconds
+[05-16 02:30:05|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002100.pt
+[05-16 02:30:05|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6313 seconds
+[05-16 02:30:05|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002100.pt
+[05-16 02:30:05|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0025 seconds
+[05-16 02:30:05|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002100.pt
+[05-16 02:30:05|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0021 seconds
+[05-16 02:30:05|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2100...
+[05-16 02:59:57|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002200.pt
+[05-16 02:59:57|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1485 seconds
+[05-16 02:59:58|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002200.pt
+[05-16 02:59:58|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6374 seconds
+[05-16 02:59:58|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002200.pt
+[05-16 02:59:58|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0033 seconds
+[05-16 02:59:58|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002200.pt
+[05-16 02:59:58|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0016 seconds
+[05-16 02:59:58|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2200...
+[05-16 03:29:50|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002300.pt
+[05-16 03:29:50|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1552 seconds
+[05-16 03:29:51|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002300.pt
+[05-16 03:29:51|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6703 seconds
+[05-16 03:29:51|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002300.pt
+[05-16 03:29:51|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0020 seconds
+[05-16 03:29:51|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002300.pt
+[05-16 03:29:51|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0014 seconds
+[05-16 03:29:51|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2300...
+[05-16 03:59:45|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002400.pt
+[05-16 03:59:45|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1591 seconds
+[05-16 03:59:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002400.pt
+[05-16 03:59:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5814 seconds
+[05-16 03:59:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002400.pt
+[05-16 03:59:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0024 seconds
+[05-16 03:59:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002400.pt
+[05-16 03:59:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0013 seconds
+[05-16 03:59:46|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2400...
+[05-16 04:29:34|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002500.pt
+[05-16 04:29:34|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1590 seconds
+[05-16 04:29:35|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002500.pt
+[05-16 04:29:35|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5715 seconds
+[05-16 04:29:35|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002500.pt
+[05-16 04:29:35|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
+[05-16 04:29:35|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002500.pt
+[05-16 04:29:35|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
+[05-16 04:29:35|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2500...
+[05-16 04:59:27|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002600.pt
+[05-16 04:59:27|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1656 seconds
+[05-16 04:59:28|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002600.pt
+[05-16 04:59:28|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6705 seconds
+[05-16 04:59:28|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002600.pt
+[05-16 04:59:28|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0016 seconds
+[05-16 04:59:28|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002600.pt
+[05-16 04:59:28|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0036 seconds
+[05-16 04:59:28|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2600...
+[05-16 05:29:18|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002700.pt
+[05-16 05:29:18|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1472 seconds
+[05-16 05:29:19|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002700.pt
+[05-16 05:29:19|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5778 seconds
+[05-16 05:29:19|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002700.pt
+[05-16 05:29:19|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0019 seconds
+[05-16 05:29:19|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002700.pt
+[05-16 05:29:19|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
+[05-16 05:29:19|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2700...
+[05-16 05:46:22|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2756...
+[05-16 06:00:01|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002800.pt
+[05-16 06:00:01|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1730 seconds
+[05-16 06:00:02|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002800.pt
+[05-16 06:00:02|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6593 seconds
+[05-16 06:00:02|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002800.pt
+[05-16 06:00:02|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0021 seconds
+[05-16 06:00:02|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002800.pt
+[05-16 06:00:02|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0016 seconds
+[05-16 06:00:02|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2800...
+[05-16 06:29:53|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002900.pt
+[05-16 06:29:53|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1591 seconds
+[05-16 06:29:54|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002900.pt
+[05-16 06:29:54|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6404 seconds
+[05-16 06:29:54|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002900.pt
+[05-16 06:29:54|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
+[05-16 06:29:54|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002900.pt
+[05-16 06:29:54|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0012 seconds
+[05-16 06:29:54|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2900...
+[05-16 06:59:44|INFO|cosmos_predict2/callbacks/iter_speed.py:80:every_n_impl] 3000 : iter_speed 17.97 seconds per iteration | Loss: 1.4185
+[05-16 06:59:44|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    66.959030      67.101379      66.816681
+peak_gpu_mem_gb               18.605011      18.611500      18.598523
+peak_gpu_mem_reserved_gb      28.186523      28.308594      28.064453
+nvml_used_gpu_mem_gb          30.944641      31.066711      30.822571
+nvml_free_gpu_mem_gb          62.639343      62.761414      62.517273
+temp                          73.500000      76.000000      71.000000
+power                     384418.500000  390913.000000  377924.000000
+util                          78.500000     100.000000      57.000000
+clock                       1612.500000    1680.000000    1545.000000
+[05-16 06:59:44|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003000.pt
+[05-16 06:59:44|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1454 seconds
+[05-16 06:59:45|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003000.pt
+[05-16 06:59:45|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5774 seconds
+[05-16 06:59:45|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003000.pt
+[05-16 06:59:45|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0024 seconds
+[05-16 06:59:45|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003000.pt
+[05-16 06:59:45|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0010 seconds
+[05-16 06:59:45|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 3000...
+[05-16 07:29:36|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003100.pt
+[05-16 07:29:36|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1724 seconds
+[05-16 07:29:37|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003100.pt
+[05-16 07:29:37|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6582 seconds
+[05-16 07:29:37|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003100.pt
+[05-16 07:29:37|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0021 seconds
+[05-16 07:29:37|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003100.pt
+[05-16 07:29:37|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0035 seconds
+[05-16 07:29:37|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 3100...
+[05-16 07:59:29|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003200.pt
+[05-16 07:59:29|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1678 seconds
+[05-16 07:59:30|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003200.pt
+[05-16 07:59:30|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6903 seconds
+[05-16 07:59:30|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003200.pt
+[05-16 07:59:30|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0024 seconds
+[05-16 07:59:30|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003200.pt
+[05-16 07:59:30|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0013 seconds
+[05-16 07:59:30|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 3200...