Upload folder using huggingface_hub

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/latest_checkpoint.txt

@@ -1 +1 @@
-iter_000001900.pt
+iter_000002000.pt

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2af8b56c1b8e724be8683b2889b14f18c3a9a1d196ea92660eee7b78ebc7e16c
+size 161817668

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95832a8c40e654ae16e1598374ebfe4b7c520f4edd78ef8fd8c6d046379786a8
+size 647025631

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca84df360cf96ab61e97368db93153f62818e5e02cde160fd6b3ee325627dbf2
+size 1602

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2b3774816544e78e539e7d51e64b43a30c795a2947d1fe5a7dbd70c48e434f7
+size 892

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/stdout.log

@@ -283,3 +283,24 @@ clock                       1650.000000    1755.000000    1545.000000
 [05-18 07:37:34|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001900.pt
 [05-18 07:37:34|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0034 seconds
 [05-18 07:37:34|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1900...
+[05-18 08:07:45|INFO|cosmos_predict2/callbacks/iter_speed.py:80:every_n_impl] 2000 : iter_speed 18.11 seconds per iteration | Loss: 1.4450
+[05-18 08:07:45|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    67.445868      67.563869      67.327866
+peak_gpu_mem_gb               18.604917      18.609647      18.600186
+peak_gpu_mem_reserved_gb      28.161133      28.279297      28.042969
+nvml_used_gpu_mem_gb          30.919250      31.037415      30.801086
+nvml_free_gpu_mem_gb          62.664734      62.782898      62.546570
+temp                          71.500000      72.000000      71.000000
+power                     377598.500000  384770.000000  370427.000000
+util                         100.000000     100.000000     100.000000
+clock                       1755.000000    1770.000000    1740.000000
+[05-18 08:07:46|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000002000.pt
+[05-18 08:07:46|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1770 seconds
+[05-18 08:07:47|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000002000.pt
+[05-18 08:07:47|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.7275 seconds
+[05-18 08:07:47|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000002000.pt
+[05-18 08:07:47|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0024 seconds
+[05-18 08:07:47|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000002000.pt
+[05-18 08:07:47|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0017 seconds
+[05-18 08:07:47|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 2000...