Upload folder using huggingface_hub

action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt

@@ -1 +1 @@
-iter_000001900.pt
+iter_000002000.pt

action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a8ed06f74357d67524d7c3a3b8e7e24aab7a3d964175ab8357a22e2b174f9ea
+size 161817668

action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68bf666d0fcbdb77340df2a380d6713bfdf7d78809d1cdb7b3f785dca4deb024
+size 647025631

action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca84df360cf96ab61e97368db93153f62818e5e02cde160fd6b3ee325627dbf2
+size 1602

action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2b3774816544e78e539e7d51e64b43a30c795a2947d1fe5a7dbd70c48e434f7
+size 892

action_decoder/chain/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/stdout.log

@@ -283,3 +283,24 @@ clock                       1687.500000    1740.000000    1635.000000
 [05-21 04:11:24|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001900.pt
 [05-21 04:11:24|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0030 seconds
 [05-21 04:11:25|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1900...
+[05-21 04:41:23|INFO|cosmos_predict2/callbacks/iter_speed.py:80:every_n_impl] 2000 : iter_speed 18.02 seconds per iteration | Loss: 1.5123
+[05-21 04:41:23|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    67.468443      67.684208      67.252678
+peak_gpu_mem_gb               18.604917      18.609647      18.600186
+peak_gpu_mem_reserved_gb      28.161133      28.279297      28.042969
+nvml_used_gpu_mem_gb          30.919250      31.037415      30.801086
+nvml_free_gpu_mem_gb          62.664734      62.782898      62.546570
+temp                          72.500000      73.000000      72.000000
+power                     381152.500000  381569.000000  380736.000000
+util                          97.000000     100.000000      94.000000
+clock                       1710.000000    1755.000000    1665.000000
+[05-21 04:41:24|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000002000.pt
+[05-21 04:41:24|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1482 seconds
+[05-21 04:41:25|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000002000.pt
+[05-21 04:41:25|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6196 seconds
+[05-21 04:41:25|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000002000.pt
+[05-21 04:41:25|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0026 seconds
+[05-21 04:41:25|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task3_1_103ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000002000.pt
+[05-21 04:41:25|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0026 seconds
+[05-21 04:41:25|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2000...