Upload folder using huggingface_hub

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/latest_checkpoint.txt

@@ -1 +1 @@
-iter_000002900.pt
+iter_000003000.pt

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97eb02a47f940bc0952de594c3bd6c00df1246f98222ea9d7c4e29a8d09ca67c
+size 161817668

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b24e09d986420c4a7d0c15ae37f1e6e7fa242413e9059b9edf46b4e7304eda49
+size 647025631

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d89fe280eaa9da2d9da6d2de29515dfd6e4c038da1bea27bbeb4ef7f3c0beebd
+size 1602

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:213554861dcd84b2c90083647dbafccb254edb3e8d462405116548b16bd548b6
+size 892

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/stdout.log

@@ -386,3 +386,24 @@ clock                       1755.000000    1770.000000    1740.000000
 [05-18 12:40:06|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002900.pt
 [05-18 12:40:06|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
 [05-18 12:40:06|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2900...
+[05-18 13:10:14|INFO|cosmos_predict2/callbacks/iter_speed.py:80:every_n_impl] 3000 : iter_speed 18.15 seconds per iteration | Loss: 2.2000
+[05-18 13:10:14|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    67.711628      67.837837      67.585419
+peak_gpu_mem_gb               18.604917      18.609647      18.600186
+peak_gpu_mem_reserved_gb      28.161133      28.279297      28.042969
+nvml_used_gpu_mem_gb          30.919250      31.037415      30.801086
+nvml_free_gpu_mem_gb          62.664734      62.782898      62.546570
+temp                          72.500000      73.000000      72.000000
+power                     385857.000000  387906.000000  383808.000000
+util                         100.000000     100.000000     100.000000
+clock                       1507.500000    1710.000000    1305.000000
+[05-18 13:10:14|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000003000.pt
+[05-18 13:10:14|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1718 seconds
+[05-18 13:10:15|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000003000.pt
+[05-18 13:10:15|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6093 seconds
+[05-18 13:10:15|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000003000.pt
+[05-18 13:10:15|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0018 seconds
+[05-18 13:10:15|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000003000.pt
+[05-18 13:10:15|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0021 seconds
+[05-18 13:10:15|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 3000...