Upload folder using huggingface_hub
Browse files
action_decoder/chain/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/stdout.log
CHANGED
|
@@ -85,3 +85,19 @@
|
|
| 85 |
[05-20 02:24:08|SUCCESS|cosmos_predict2/pipelines/video2world.py:354:from_config] Successfully loaded DiT from /home/shadeform/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt
|
| 86 |
[05-20 02:24:09|INFO|cosmos_predict2/models/world2action_model.py:148:__init__] Total parameters: 2.04B, Frozen parameters: 1,956,413,440, Trainable parameters: 80,859,000
|
| 87 |
[05-20 02:24:09|INFO|cosmos_predict2/models/world2action_model.py:167:__init__] FSDP (Fully Sharded Data Parallel) is disabled.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
[05-20 02:24:08|SUCCESS|cosmos_predict2/pipelines/video2world.py:354:from_config] Successfully loaded DiT from /home/shadeform/pdt-mimic/mimic-video/model/checkpoints/video_backbone/v2w_push_combined_lora_rank32_lr1.778e-04_bsz32_iter_000002500_fused.pt
|
| 86 |
[05-20 02:24:09|INFO|cosmos_predict2/models/world2action_model.py:148:__init__] Total parameters: 2.04B, Frozen parameters: 1,956,413,440, Trainable parameters: 80,859,000
|
| 87 |
[05-20 02:24:09|INFO|cosmos_predict2/models/world2action_model.py:167:__init__] FSDP (Fully Sharded Data Parallel) is disabled.
|
| 88 |
+
[05-20 03:01:27|CRITICAL|cosmos_predict2/utils/optim_instantiate_dtensor.py:49:get_base_optimizer] total num parameters : 80,859,000
|
| 89 |
+
[05-20 03:01:27|WARNING|cosmos_predict2/utils/fused_adam_dtensor.py:103:__init__] FusedAdam master_weights: True capturable: True
|
| 90 |
+
[05-20 03:01:27|INFO|cosmos_predict2/checkpointer.py:288:load] Training from scratch.
|
| 91 |
+
[05-20 03:01:27|INFO|imaginaire/utils/misc.py:266:wrapper] Time spent on checkpoint loading: 0.0004 seconds
|
| 92 |
+
[05-20 03:01:27|CRITICAL|imaginaire/trainer.py:179:train] Distributed parallelism mode: ddp
|
| 93 |
+
[05-20 03:01:27|INFO|imaginaire/trainer.py:187:train] Starting training...
|
| 94 |
+
[05-20 03:01:27|INFO|cosmos_predict2/callbacks/device_monitor.py:92:on_train_start] DeviceMonitor callback: local_dir: checkpoints/vam/so101/w2a_so101_task2_slow_136ep_lora32_iter2500_75m_2gpu/DeviceMonitor
|
| 95 |
+
[05-20 03:01:29|INFO|imaginaire/trainer.py:388:validate] Validating at iteration 0...
|
| 96 |
+
[05-20 03:02:10|INFO|imaginaire/trainer.py:193:train] Initial validation done.
|
| 97 |
+
[05-20 03:02:17|WARNING|imaginaire/utils/distributed.py:284:ddp_sync_grad] DDP static_graph=True is incompatible with sync_grad(). Performance will be reduced.
|
| 98 |
+
[05-20 03:02:54|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 1: Hit counter: 1/5 | Loss: 21.0455 | Time: 2336.59s
|
| 99 |
+
[05-20 03:03:14|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 2: Hit counter: 2/5 | Loss: 33.0988 | Time: 19.60s
|
| 100 |
+
[05-20 03:03:33|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 3: Hit counter: 3/5 | Loss: 19.2938 | Time: 19.66s
|
| 101 |
+
[05-20 03:03:53|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 4: Hit counter: 4/5 | Loss: 59.5364 | Time: 19.69s
|
| 102 |
+
[05-20 03:04:13|INFO|cosmos_predict2/callbacks/iter_speed.py:51:on_training_step_end] Iteration 5: Hit counter: 5/5 | Loss: 36.8914 | Time: 19.76s
|
| 103 |
+
[05-20 03:10:50|CRITICAL|imaginaire/callbacks/manual_gc.py:48:every_n_impl] Garbage collection disabled
|