Upload folder using huggingface_hub

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt

@@ -1 +1 @@
-iter_000001900.pt
+iter_000002000.pt

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ec0c3f874b9a199dd4569819a974091995e60007a54bce1c9141962ad7fee75
+size 161817668

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebb7411791691f60c4bb93f08fd4177a62eeb65da5d83b6db68f05108f6fa0d4
+size 647025631

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca84df360cf96ab61e97368db93153f62818e5e02cde160fd6b3ee325627dbf2
+size 1602

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2b3774816544e78e539e7d51e64b43a30c795a2947d1fe5a7dbd70c48e434f7
+size 892

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/stdout.log

@@ -283,3 +283,24 @@ clock                       1567.500000    1575.000000    1560.000000
 [05-20 13:42:07|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001900.pt
 [05-20 13:42:07|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0013 seconds
 [05-20 13:42:07|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1900...
+[05-20 14:15:55|INFO|cosmos_predict2/callbacks/iter_speed.py:80:every_n_impl] 2000 : iter_speed 20.20 seconds per iteration | Loss: 1.9108
+[05-20 14:15:55|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    80.627443      84.234562      77.020325
+peak_gpu_mem_gb               18.603743      18.610725      18.596760
+peak_gpu_mem_reserved_gb      27.959961      27.978516      27.941406
+nvml_used_gpu_mem_gb          29.467285      29.485840      29.448730
+nvml_free_gpu_mem_gb          50.180176      50.198730      50.161621
+temp                          69.500000      72.000000      67.000000
+power                     338781.000000  339882.000000  337680.000000
+util                         100.000000     100.000000     100.000000
+clock                       1455.000000    1500.000000    1410.000000
+[05-20 14:15:55|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002000.pt
+[05-20 14:15:55|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1479 seconds
+[05-20 14:15:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002000.pt
+[05-20 14:15:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5839 seconds
+[05-20 14:15:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000002000.pt
+[05-20 14:15:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0014 seconds
+[05-20 14:15:56|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002000.pt
+[05-20 14:15:56|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0008 seconds
+[05-20 14:15:56|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2000...