Upload folder using huggingface_hub

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/latest_checkpoint.txt

@@ -1 +1 @@
-iter_000000900.pt
+iter_000001000.pt

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a0ff5430c7b61ecde833f9ce5703ab60d319dffc475ae6a0b92c9d7aad23fcc
+size 161817668

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a821fb6b4c3a3314c0b382eff645f0d58a8af8f7ae17113ba8e47da8a4e3d09
+size 647025631

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47a6bae9e822d2753aee6b0b62d6f98bb46ade9ec4e41da26490e0201d09c172
+size 1602

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a8eb54f575f232d4f03d7c9594005cb06ae9ffb4bc249977d1154493cfbf75
+size 892

action_decoder/chain/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/stdout.log

@@ -182,3 +182,23 @@
 [05-18 02:35:38|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000000900.pt
 [05-18 02:35:38|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0012 seconds
 [05-18 02:35:38|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
+[05-18 03:05:53|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    67.678272      67.734711      67.621834
+peak_gpu_mem_gb               18.604917      18.609647      18.600186
+peak_gpu_mem_reserved_gb      28.161133      28.279297      28.042969
+nvml_used_gpu_mem_gb          30.919250      31.037415      30.801086
+nvml_free_gpu_mem_gb          62.664734      62.782898      62.546570
+temp                          71.000000      72.000000      70.000000
+power                     382072.000000  387818.000000  376326.000000
+util                         100.000000     100.000000     100.000000
+clock                       1650.000000    1755.000000    1545.000000
+[05-18 03:05:53|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/model/iter_000001000.pt
+[05-18 03:05:53|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1776 seconds
+[05-18 03:05:54|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/optim/iter_000001000.pt
+[05-18 03:05:54|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.6632 seconds
+[05-18 03:05:54|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/scheduler/iter_000001000.pt
+[05-18 03:05:54|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0022 seconds
+[05-18 03:05:54|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_combined_lora32_iter2500_75m_2gpu_bsz2_accum8_val5/checkpoints/trainer/iter_000001000.pt
+[05-18 03:05:54|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0040 seconds
+[05-18 03:05:54|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1000...