Upload folder using huggingface_hub

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/latest_checkpoint.txt

@@ -1 +1 @@
-iter_000000900.pt
+iter_000001000.pt

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b4e4b07d9a3c757289357e3b41e0ae71e62360ce04974eefead29a53e1498f8
+size 161817668

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:822d754b45942f93a39ee002b70f5b4590b70c72a0fdc8790a0b2e4cc2914345
+size 647025631

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47a6bae9e822d2753aee6b0b62d6f98bb46ade9ec4e41da26490e0201d09c172
+size 1602

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a8eb54f575f232d4f03d7c9594005cb06ae9ffb4bc249977d1154493cfbf75
+size 892

action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/stdout.log

@@ -182,3 +182,23 @@
 [05-20 08:05:31|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000000900.pt
 [05-20 08:05:31|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0009 seconds
 [05-20 08:05:31|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 900...
+[05-20 08:39:11|INFO|cosmos_predict2/callbacks/device_monitor.py:145:every_n_impl] DeviceMonitor Stats:
+                                    Avg            Max            Min
+cpu_mem_gb                    80.738853      84.667767      76.809940
+peak_gpu_mem_gb               18.603743      18.610725      18.596760
+peak_gpu_mem_reserved_gb      27.959961      27.978516      27.941406
+nvml_used_gpu_mem_gb          29.467285      29.485840      29.448730
+nvml_free_gpu_mem_gb          50.180176      50.198730      50.161621
+temp                          69.000000      72.000000      66.000000
+power                     337843.000000  338515.000000  337171.000000
+util                          97.500000     100.000000      95.000000
+clock                       1567.500000    1575.000000    1560.000000
+[05-20 08:39:12|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/model/iter_000001000.pt
+[05-20 08:39:12|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.1491 seconds
+[05-20 08:39:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/optim/iter_000001000.pt
+[05-20 08:39:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.5753 seconds
+[05-20 08:39:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/scheduler/iter_000001000.pt
+[05-20 08:39:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0014 seconds
+[05-20 08:39:13|SUCCESS|cosmos_predict2/checkpointer.py:139:_save_worker_local] Saved checkpoint (local): checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/checkpoints/trainer/iter_000001000.pt
+[05-20 08:39:13|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint saving (local): 0.0015 seconds
+[05-20 08:39:13|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 1000...