Upload folder using huggingface_hub
Browse files- .gitattributes +3 -0
- fastumi_pickandplace_qwenPI_329v4/checkpoints/steps_15000_pytorch_model.pt +3 -0
- fastumi_pickandplace_qwenPI_329v4/config.yaml +1 -1
- fastumi_pickandplace_qwenPI_329v4/summary.jsonl +1 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/debug-internal.log +7 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/debug.log +0 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/config.yaml +172 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/output.log +0 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/requirements.txt +161 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/wandb-metadata.json +161 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/wandb-summary.json +1 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/logs/debug-core.log +19 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/logs/debug-internal.log +12 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/logs/debug.log +0 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/run-q6i4v9a9.wandb +3 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/config.yaml +171 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/output.log +77 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/requirements.txt +161 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/wandb-metadata.json +161 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/wandb-summary.json +1 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/logs/debug-core.log +13 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/logs/debug-internal.log +8 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/logs/debug.log +0 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/run-du1a557y.wandb +3 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/config.yaml +171 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/output.log +317 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/requirements.txt +161 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/wandb-metadata.json +161 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/wandb-summary.json +1 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/logs/debug-core.log +13 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/logs/debug-internal.log +7 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/logs/debug.log +0 -0
- fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/run-n5d66d57.wandb +3 -0
.gitattributes
CHANGED
|
@@ -68,3 +68,6 @@ fastumi_pickandplace_qwenDiscreteDiffusion_329v4/openloop_eval_steps_20000_pos_s
|
|
| 68 |
fastumi_pickandplace_qwenDiscreteDiffusion_329v4/wandb/wandb/run-20260402_015347-rztzqreo/run-rztzqreo.wandb filter=lfs diff=lfs merge=lfs -text
|
| 69 |
fastumi_pickandplace_qwenDiscreteDiffusion_329v4/wandb/wandb/run-20260402_174446-b7f4uglx/run-b7f4uglx.wandb filter=lfs diff=lfs merge=lfs -text
|
| 70 |
fastumi_pickandplace_qwenDiscreteDiffusion_329v4/wandb/wandb/run-20260402_182806-6dhdda09/run-6dhdda09.wandb filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
fastumi_pickandplace_qwenDiscreteDiffusion_329v4/wandb/wandb/run-20260402_015347-rztzqreo/run-rztzqreo.wandb filter=lfs diff=lfs merge=lfs -text
|
| 69 |
fastumi_pickandplace_qwenDiscreteDiffusion_329v4/wandb/wandb/run-20260402_174446-b7f4uglx/run-b7f4uglx.wandb filter=lfs diff=lfs merge=lfs -text
|
| 70 |
fastumi_pickandplace_qwenDiscreteDiffusion_329v4/wandb/wandb/run-20260402_182806-6dhdda09/run-6dhdda09.wandb filter=lfs diff=lfs merge=lfs -text
|
| 71 |
+
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/run-q6i4v9a9.wandb filter=lfs diff=lfs merge=lfs -text
|
| 72 |
+
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/run-du1a557y.wandb filter=lfs diff=lfs merge=lfs -text
|
| 73 |
+
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/run-n5d66d57.wandb filter=lfs diff=lfs merge=lfs -text
|
fastumi_pickandplace_qwenPI_329v4/checkpoints/steps_15000_pytorch_model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:79eb78937677daa83f9b51f15304831aac558f65dc5a4e9128aba2e435ae82de
|
| 3 |
+
size 12444522384
|
fastumi_pickandplace_qwenPI_329v4/config.yaml
CHANGED
|
@@ -55,7 +55,7 @@ trainer:
|
|
| 55 |
qwen_vl_interface: 1.0e-05
|
| 56 |
logging_frequency: 50
|
| 57 |
lr_scheduler_type: cosine_with_min_lr
|
| 58 |
-
max_train_steps:
|
| 59 |
num_warmup_steps: 5000
|
| 60 |
optimizer:
|
| 61 |
betas:
|
|
|
|
| 55 |
qwen_vl_interface: 1.0e-05
|
| 56 |
logging_frequency: 50
|
| 57 |
lr_scheduler_type: cosine_with_min_lr
|
| 58 |
+
max_train_steps: 20000
|
| 59 |
num_warmup_steps: 5000
|
| 60 |
optimizer:
|
| 61 |
betas:
|
fastumi_pickandplace_qwenPI_329v4/summary.jsonl
CHANGED
|
@@ -1,2 +1,3 @@
|
|
| 1 |
{"steps": 5000}
|
| 2 |
{"steps": 10000}
|
|
|
|
|
|
| 1 |
{"steps": 5000}
|
| 2 |
{"steps": 10000}
|
| 3 |
+
{"steps": 15000}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/debug-internal.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-03T01:45:17.263943855Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
|
| 2 |
+
{"time":"2026-04-03T01:45:17.502454295Z","level":"INFO","msg":"stream: created new stream","id":"n5d66d57"}
|
| 3 |
+
{"time":"2026-04-03T01:45:17.502549217Z","level":"INFO","msg":"handler: started","stream_id":"n5d66d57"}
|
| 4 |
+
{"time":"2026-04-03T01:45:17.50265022Z","level":"INFO","msg":"stream: started","id":"n5d66d57"}
|
| 5 |
+
{"time":"2026-04-03T01:45:17.50266136Z","level":"INFO","msg":"writer: started","stream_id":"n5d66d57"}
|
| 6 |
+
{"time":"2026-04-03T01:45:17.50267484Z","level":"INFO","msg":"sender: started","stream_id":"n5d66d57"}
|
| 7 |
+
{"time":"2026-04-03T05:15:08.203590884Z","level":"INFO","msg":"stream: closing","id":"n5d66d57"}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/debug.log
ADDED
|
File without changes
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/config.yaml
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.0
|
| 4 |
+
e:
|
| 5 |
+
dd3cdsyzuoci9p4000nty15269eqr8z0:
|
| 6 |
+
args:
|
| 7 |
+
- --config_yaml
|
| 8 |
+
- ./examples/calvin/train_files/starvla_train_calvin.yaml
|
| 9 |
+
- --framework.name
|
| 10 |
+
- QwenPI
|
| 11 |
+
- --framework.qwenvl.base_vlm
|
| 12 |
+
- playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action
|
| 13 |
+
- --framework.qwenvl.attn_implementation
|
| 14 |
+
- flash_attention_2
|
| 15 |
+
- --framework.action_model.action_dim
|
| 16 |
+
- "10"
|
| 17 |
+
- --framework.action_model.state_dim
|
| 18 |
+
- "10"
|
| 19 |
+
- --framework.action_model.future_action_window_size
|
| 20 |
+
- "15"
|
| 21 |
+
- --framework.action_model.past_action_window_size
|
| 22 |
+
- "0"
|
| 23 |
+
- --framework.action_model.action_hidden_dim
|
| 24 |
+
- "1024"
|
| 25 |
+
- --framework.action_model.hidden_size
|
| 26 |
+
- "1024"
|
| 27 |
+
- --framework.action_model.action_model_type
|
| 28 |
+
- DiT-B
|
| 29 |
+
- --framework.action_model.add_pos_embed
|
| 30 |
+
- "True"
|
| 31 |
+
- --framework.action_model.max_seq_len
|
| 32 |
+
- "1024"
|
| 33 |
+
- --framework.action_model.noise_beta_alpha
|
| 34 |
+
- "1.5"
|
| 35 |
+
- --framework.action_model.noise_beta_beta
|
| 36 |
+
- "1.0"
|
| 37 |
+
- --framework.action_model.noise_s
|
| 38 |
+
- "0.999"
|
| 39 |
+
- --framework.action_model.num_timestep_buckets
|
| 40 |
+
- "1000"
|
| 41 |
+
- --framework.action_model.num_inference_timesteps
|
| 42 |
+
- "4"
|
| 43 |
+
- --framework.action_model.num_target_vision_tokens
|
| 44 |
+
- "32"
|
| 45 |
+
- --datasets.vla_data.data_root_dir
|
| 46 |
+
- playground/Datasets/FastUMI
|
| 47 |
+
- --datasets.vla_data.data_mix
|
| 48 |
+
- dynamic-329-v4
|
| 49 |
+
- --datasets.vla_data.include_state
|
| 50 |
+
- "false"
|
| 51 |
+
- --datasets.vla_data.per_device_batch_size
|
| 52 |
+
- "8"
|
| 53 |
+
- --datasets.vla_data.video_backend
|
| 54 |
+
- torchvision_av
|
| 55 |
+
- --trainer.freeze_modules
|
| 56 |
+
- ""
|
| 57 |
+
- --trainer.max_train_steps
|
| 58 |
+
- "10000"
|
| 59 |
+
- --trainer.save_interval
|
| 60 |
+
- "5000"
|
| 61 |
+
- --trainer.logging_frequency
|
| 62 |
+
- "50"
|
| 63 |
+
- --trainer.eval_interval
|
| 64 |
+
- "100"
|
| 65 |
+
- --trainer.gradient_accumulation_steps
|
| 66 |
+
- "1"
|
| 67 |
+
- --trainer.is_resume
|
| 68 |
+
- "true"
|
| 69 |
+
- --run_root_dir
|
| 70 |
+
- ./results/Checkpoints
|
| 71 |
+
- --run_id
|
| 72 |
+
- fastumi_pickandplace_qwenPI_329v4
|
| 73 |
+
- --wandb_project
|
| 74 |
+
- starVLA_FastUMI_dynamic_1
|
| 75 |
+
- --wandb_entity
|
| 76 |
+
- 2200011093-peking-university
|
| 77 |
+
codePath: starVLA/training/train_starvla.py
|
| 78 |
+
codePathLocal: starVLA/training/train_starvla.py
|
| 79 |
+
cpu_count: 192
|
| 80 |
+
cpu_count_logical: 384
|
| 81 |
+
cudaVersion: "13.1"
|
| 82 |
+
disk:
|
| 83 |
+
/:
|
| 84 |
+
total: "3776651378688"
|
| 85 |
+
used: "158830641152"
|
| 86 |
+
email: wangpc@berkeley.edu
|
| 87 |
+
executable: /home/wangpc/miniconda3/envs/starVLA/bin/python3.10
|
| 88 |
+
git:
|
| 89 |
+
commit: 15c7d05ddb9af57d0a343b79a1bce9cf8b59b9a5
|
| 90 |
+
remote: https://github.com/Kaiwen-Hong/starVLA.git
|
| 91 |
+
gpu: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 92 |
+
gpu_count: 8
|
| 93 |
+
gpu_nvidia:
|
| 94 |
+
- architecture: Blackwell
|
| 95 |
+
cudaCores: 24064
|
| 96 |
+
memoryTotal: "102641958912"
|
| 97 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 98 |
+
uuid: GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673
|
| 99 |
+
- architecture: Blackwell
|
| 100 |
+
cudaCores: 24064
|
| 101 |
+
memoryTotal: "102641958912"
|
| 102 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 103 |
+
uuid: GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5
|
| 104 |
+
- architecture: Blackwell
|
| 105 |
+
cudaCores: 24064
|
| 106 |
+
memoryTotal: "102641958912"
|
| 107 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 108 |
+
uuid: GPU-1f3c0889-b740-5143-e064-afeb49245756
|
| 109 |
+
- architecture: Blackwell
|
| 110 |
+
cudaCores: 24064
|
| 111 |
+
memoryTotal: "102641958912"
|
| 112 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 113 |
+
uuid: GPU-49955a45-509a-e8af-a468-b9ef0449b005
|
| 114 |
+
- architecture: Blackwell
|
| 115 |
+
cudaCores: 24064
|
| 116 |
+
memoryTotal: "102641958912"
|
| 117 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 118 |
+
uuid: GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd
|
| 119 |
+
- architecture: Blackwell
|
| 120 |
+
cudaCores: 24064
|
| 121 |
+
memoryTotal: "102641958912"
|
| 122 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 123 |
+
uuid: GPU-d16b5acf-a100-2d1b-8138-1661892f3d26
|
| 124 |
+
- architecture: Blackwell
|
| 125 |
+
cudaCores: 24064
|
| 126 |
+
memoryTotal: "102641958912"
|
| 127 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 128 |
+
uuid: GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab
|
| 129 |
+
- architecture: Blackwell
|
| 130 |
+
cudaCores: 24064
|
| 131 |
+
memoryTotal: "102641958912"
|
| 132 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 133 |
+
uuid: GPU-747dc32b-b025-76e9-697d-fd33ef441b47
|
| 134 |
+
host: tams02
|
| 135 |
+
memory:
|
| 136 |
+
total: "1081552064512"
|
| 137 |
+
os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
|
| 138 |
+
program: /scratch/wangpc/starVLA/starVLA/training/train_starvla.py
|
| 139 |
+
python: CPython 3.10.19
|
| 140 |
+
root: ./results/Checkpoints/fastumi_pickandplace_qwenPI_329v4/wandb
|
| 141 |
+
startedAt: "2026-04-02T07:25:18.492408Z"
|
| 142 |
+
writerId: dd3cdsyzuoci9p4000nty15269eqr8z0
|
| 143 |
+
m: []
|
| 144 |
+
python_version: 3.10.19
|
| 145 |
+
t:
|
| 146 |
+
"1":
|
| 147 |
+
- 1
|
| 148 |
+
- 11
|
| 149 |
+
- 41
|
| 150 |
+
- 49
|
| 151 |
+
- 63
|
| 152 |
+
- 71
|
| 153 |
+
- 80
|
| 154 |
+
- 83
|
| 155 |
+
"2":
|
| 156 |
+
- 1
|
| 157 |
+
- 11
|
| 158 |
+
- 41
|
| 159 |
+
- 49
|
| 160 |
+
- 63
|
| 161 |
+
- 71
|
| 162 |
+
- 80
|
| 163 |
+
- 83
|
| 164 |
+
"3":
|
| 165 |
+
- 2
|
| 166 |
+
- 13
|
| 167 |
+
- 61
|
| 168 |
+
"4": 3.10.19
|
| 169 |
+
"5": 0.25.0
|
| 170 |
+
"6": 4.57.0
|
| 171 |
+
"12": 0.25.0
|
| 172 |
+
"13": linux-x86_64
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/requirements.txt
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
starVLA==1.0.1
|
| 2 |
+
kiwisolver==1.4.9
|
| 3 |
+
scipy==1.15.3
|
| 4 |
+
pyarrow==14.0.1
|
| 5 |
+
protobuf==6.33.5
|
| 6 |
+
platformdirs==4.9.4
|
| 7 |
+
mdurl==0.1.2
|
| 8 |
+
Jinja2==3.1.6
|
| 9 |
+
torchvision==0.22.0+cu128
|
| 10 |
+
exceptiongroup==1.3.1
|
| 11 |
+
nvidia-cusparselt-cu12==0.6.3
|
| 12 |
+
markdown-it-py==4.0.0
|
| 13 |
+
ray==2.54.0
|
| 14 |
+
timm==1.0.25
|
| 15 |
+
nvidia-nvjitlink-cu12==12.8.61
|
| 16 |
+
urllib3==2.6.3
|
| 17 |
+
numpydantic==1.6.9
|
| 18 |
+
pillow==12.1.1
|
| 19 |
+
json-numpy==2.1.1
|
| 20 |
+
fastparquet==2024.11.0
|
| 21 |
+
contourpy==1.3.2
|
| 22 |
+
tensorboard-data-server==0.7.2
|
| 23 |
+
albumentations==1.4.18
|
| 24 |
+
deepspeed==0.16.9
|
| 25 |
+
ImageIO==2.37.2
|
| 26 |
+
huggingface_hub==0.36.2
|
| 27 |
+
hjson==3.1.0
|
| 28 |
+
tqdm==4.67.3
|
| 29 |
+
idna==3.11
|
| 30 |
+
packaging==25.0
|
| 31 |
+
python-dateutil==2.9.0.post0
|
| 32 |
+
annotated-types==0.7.0
|
| 33 |
+
regex==2026.2.28
|
| 34 |
+
jsonschema-specifications==2025.9.1
|
| 35 |
+
snntorch==0.9.4
|
| 36 |
+
rpds-py==0.30.0
|
| 37 |
+
cramjam==2.11.0
|
| 38 |
+
importlib_metadata==8.7.1
|
| 39 |
+
torch==2.7.0+cu128
|
| 40 |
+
yacs==0.1.8
|
| 41 |
+
msgpack==1.1.2
|
| 42 |
+
h11==0.16.0
|
| 43 |
+
nvidia-cuda-runtime-cu12==12.8.57
|
| 44 |
+
typing_extensions==4.15.0
|
| 45 |
+
scikit-image==0.25.2
|
| 46 |
+
mpmath==1.3.0
|
| 47 |
+
einops==0.8.2
|
| 48 |
+
wandb==0.25.0
|
| 49 |
+
anyio==4.12.1
|
| 50 |
+
flash_attn==2.7.4.post1
|
| 51 |
+
websockets==15.0.1
|
| 52 |
+
requests==2.32.5
|
| 53 |
+
accelerate==1.5.2
|
| 54 |
+
nvidia-cuda-cupti-cu12==12.8.57
|
| 55 |
+
nvidia-cuda-nvrtc-cu12==12.8.61
|
| 56 |
+
PyYAML==6.0.3
|
| 57 |
+
absl-py==2.4.0
|
| 58 |
+
httpx==0.28.1
|
| 59 |
+
pipablepytorch3d==0.7.6
|
| 60 |
+
nvidia-cublas-cu12==12.8.3.14
|
| 61 |
+
PyMuPDF==1.27.2.2
|
| 62 |
+
decord==0.6.0
|
| 63 |
+
ninja==1.13.0
|
| 64 |
+
albucore==0.0.17
|
| 65 |
+
attrs==26.1.0
|
| 66 |
+
pyparsing==3.3.2
|
| 67 |
+
triton==3.3.0
|
| 68 |
+
Markdown==3.10.2
|
| 69 |
+
Pygments==2.19.2
|
| 70 |
+
pydantic==2.10.6
|
| 71 |
+
tabulate==0.10.0
|
| 72 |
+
termcolor==3.3.0
|
| 73 |
+
zipp==3.23.0
|
| 74 |
+
Werkzeug==3.1.6
|
| 75 |
+
sympy==1.14.0
|
| 76 |
+
debugpy==1.8.20
|
| 77 |
+
certifi==2026.2.25
|
| 78 |
+
websocket==0.2.1
|
| 79 |
+
fonttools==4.61.1
|
| 80 |
+
transformers==4.57.0
|
| 81 |
+
av==12.3.0
|
| 82 |
+
transformers-stream-generator==0.0.4
|
| 83 |
+
nvidia-cudnn-cu12==9.7.1.26
|
| 84 |
+
nvidia-curand-cu12==10.3.9.55
|
| 85 |
+
six==1.17.0
|
| 86 |
+
fvcore==0.1.5.post20221221
|
| 87 |
+
matplotlib==3.10.8
|
| 88 |
+
lazy_loader==0.4
|
| 89 |
+
nvidia-nccl-cu12==2.26.2
|
| 90 |
+
diffusers==0.37.0
|
| 91 |
+
tifffile==2025.5.10
|
| 92 |
+
GitPython==3.1.46
|
| 93 |
+
tokenizers==0.22.2
|
| 94 |
+
eval_type_backport==0.3.1
|
| 95 |
+
nvidia-cufile-cu12==1.13.0.11
|
| 96 |
+
numpy==1.26.4
|
| 97 |
+
filelock==3.25.0
|
| 98 |
+
fsspec==2026.2.0
|
| 99 |
+
nvidia-cusolver-cu12==11.7.2.55
|
| 100 |
+
MarkupSafe==3.0.3
|
| 101 |
+
tyro==1.0.8
|
| 102 |
+
pydantic_core==2.27.2
|
| 103 |
+
portalocker==3.2.0
|
| 104 |
+
qwen-vl-utils==0.0.14
|
| 105 |
+
click==8.3.1
|
| 106 |
+
tiktoken==0.12.0
|
| 107 |
+
smmap==5.0.2
|
| 108 |
+
nvidia-nvtx-cu12==12.8.55
|
| 109 |
+
pytz==2026.1.post1
|
| 110 |
+
rich==14.2.0
|
| 111 |
+
charset-normalizer==3.4.4
|
| 112 |
+
tensorboard==2.20.0
|
| 113 |
+
zope.event==6.1
|
| 114 |
+
zope.interface==8.2
|
| 115 |
+
networkx==3.4.2
|
| 116 |
+
mpi4py==4.1.1
|
| 117 |
+
gitdb==4.0.12
|
| 118 |
+
safetensors==0.7.0
|
| 119 |
+
typeguard==4.5.1
|
| 120 |
+
py-cpuinfo==9.0.0
|
| 121 |
+
websocket-client==1.8.0
|
| 122 |
+
hf-xet==1.3.2
|
| 123 |
+
wheel==0.46.3
|
| 124 |
+
jsonschema==4.26.0
|
| 125 |
+
antlr4-python3-runtime==4.9.3
|
| 126 |
+
gevent==25.9.1
|
| 127 |
+
setuptools==80.9.0
|
| 128 |
+
torchaudio==2.7.0+cu128
|
| 129 |
+
nvidia-cufft-cu12==11.3.3.41
|
| 130 |
+
greenlet==3.3.2
|
| 131 |
+
docstring_parser==0.17.0
|
| 132 |
+
iopath==0.1.10
|
| 133 |
+
cycler==0.12.1
|
| 134 |
+
tzdata==2025.3
|
| 135 |
+
psutil==7.2.2
|
| 136 |
+
referencing==0.37.0
|
| 137 |
+
grpcio==1.78.0
|
| 138 |
+
nvidia-cusparse-cu12==12.5.7.53
|
| 139 |
+
sentry-sdk==2.54.0
|
| 140 |
+
httpcore==1.0.9
|
| 141 |
+
opencv-python-headless==4.11.0.86
|
| 142 |
+
omegaconf==2.3.0
|
| 143 |
+
pandas==2.3.3
|
| 144 |
+
eva-decord==0.6.1
|
| 145 |
+
pip==26.0.1
|
| 146 |
+
inflect==7.3.1
|
| 147 |
+
jaraco.text==3.12.1
|
| 148 |
+
autocommand==2.2.2
|
| 149 |
+
backports.tarfile==1.2.0
|
| 150 |
+
jaraco.functools==4.0.1
|
| 151 |
+
zipp==3.19.2
|
| 152 |
+
more-itertools==10.3.0
|
| 153 |
+
tomli==2.0.1
|
| 154 |
+
typing_extensions==4.12.2
|
| 155 |
+
typeguard==4.3.0
|
| 156 |
+
wheel==0.45.1
|
| 157 |
+
platformdirs==4.2.2
|
| 158 |
+
importlib_metadata==8.0.0
|
| 159 |
+
jaraco.collections==5.1.0
|
| 160 |
+
packaging==24.2
|
| 161 |
+
jaraco.context==5.3.0
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-106-generic-x86_64-with-glibc2.39",
|
| 3 |
+
"python": "CPython 3.10.19",
|
| 4 |
+
"startedAt": "2026-04-02T07:25:18.492408Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config_yaml",
|
| 7 |
+
"./examples/calvin/train_files/starvla_train_calvin.yaml",
|
| 8 |
+
"--framework.name",
|
| 9 |
+
"QwenPI",
|
| 10 |
+
"--framework.qwenvl.base_vlm",
|
| 11 |
+
"playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action",
|
| 12 |
+
"--framework.qwenvl.attn_implementation",
|
| 13 |
+
"flash_attention_2",
|
| 14 |
+
"--framework.action_model.action_dim",
|
| 15 |
+
"10",
|
| 16 |
+
"--framework.action_model.state_dim",
|
| 17 |
+
"10",
|
| 18 |
+
"--framework.action_model.future_action_window_size",
|
| 19 |
+
"15",
|
| 20 |
+
"--framework.action_model.past_action_window_size",
|
| 21 |
+
"0",
|
| 22 |
+
"--framework.action_model.action_hidden_dim",
|
| 23 |
+
"1024",
|
| 24 |
+
"--framework.action_model.hidden_size",
|
| 25 |
+
"1024",
|
| 26 |
+
"--framework.action_model.action_model_type",
|
| 27 |
+
"DiT-B",
|
| 28 |
+
"--framework.action_model.add_pos_embed",
|
| 29 |
+
"True",
|
| 30 |
+
"--framework.action_model.max_seq_len",
|
| 31 |
+
"1024",
|
| 32 |
+
"--framework.action_model.noise_beta_alpha",
|
| 33 |
+
"1.5",
|
| 34 |
+
"--framework.action_model.noise_beta_beta",
|
| 35 |
+
"1.0",
|
| 36 |
+
"--framework.action_model.noise_s",
|
| 37 |
+
"0.999",
|
| 38 |
+
"--framework.action_model.num_timestep_buckets",
|
| 39 |
+
"1000",
|
| 40 |
+
"--framework.action_model.num_inference_timesteps",
|
| 41 |
+
"4",
|
| 42 |
+
"--framework.action_model.num_target_vision_tokens",
|
| 43 |
+
"32",
|
| 44 |
+
"--datasets.vla_data.data_root_dir",
|
| 45 |
+
"playground/Datasets/FastUMI",
|
| 46 |
+
"--datasets.vla_data.data_mix",
|
| 47 |
+
"dynamic-329-v4",
|
| 48 |
+
"--datasets.vla_data.include_state",
|
| 49 |
+
"false",
|
| 50 |
+
"--datasets.vla_data.per_device_batch_size",
|
| 51 |
+
"8",
|
| 52 |
+
"--datasets.vla_data.video_backend",
|
| 53 |
+
"torchvision_av",
|
| 54 |
+
"--trainer.freeze_modules",
|
| 55 |
+
"",
|
| 56 |
+
"--trainer.max_train_steps",
|
| 57 |
+
"10000",
|
| 58 |
+
"--trainer.save_interval",
|
| 59 |
+
"5000",
|
| 60 |
+
"--trainer.logging_frequency",
|
| 61 |
+
"50",
|
| 62 |
+
"--trainer.eval_interval",
|
| 63 |
+
"100",
|
| 64 |
+
"--trainer.gradient_accumulation_steps",
|
| 65 |
+
"1",
|
| 66 |
+
"--trainer.is_resume",
|
| 67 |
+
"true",
|
| 68 |
+
"--run_root_dir",
|
| 69 |
+
"./results/Checkpoints",
|
| 70 |
+
"--run_id",
|
| 71 |
+
"fastumi_pickandplace_qwenPI_329v4",
|
| 72 |
+
"--wandb_project",
|
| 73 |
+
"starVLA_FastUMI_dynamic_1",
|
| 74 |
+
"--wandb_entity",
|
| 75 |
+
"2200011093-peking-university"
|
| 76 |
+
],
|
| 77 |
+
"program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py",
|
| 78 |
+
"codePath": "starVLA/training/train_starvla.py",
|
| 79 |
+
"codePathLocal": "starVLA/training/train_starvla.py",
|
| 80 |
+
"git": {
|
| 81 |
+
"remote": "https://github.com/Kaiwen-Hong/starVLA.git",
|
| 82 |
+
"commit": "15c7d05ddb9af57d0a343b79a1bce9cf8b59b9a5"
|
| 83 |
+
},
|
| 84 |
+
"email": "wangpc@berkeley.edu",
|
| 85 |
+
"root": "./results/Checkpoints/fastumi_pickandplace_qwenPI_329v4/wandb",
|
| 86 |
+
"host": "tams02",
|
| 87 |
+
"executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10",
|
| 88 |
+
"cpu_count": 192,
|
| 89 |
+
"cpu_count_logical": 384,
|
| 90 |
+
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 91 |
+
"gpu_count": 8,
|
| 92 |
+
"disk": {
|
| 93 |
+
"/": {
|
| 94 |
+
"total": "3776651378688",
|
| 95 |
+
"used": "158830641152"
|
| 96 |
+
}
|
| 97 |
+
},
|
| 98 |
+
"memory": {
|
| 99 |
+
"total": "1081552064512"
|
| 100 |
+
},
|
| 101 |
+
"gpu_nvidia": [
|
| 102 |
+
{
|
| 103 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 104 |
+
"memoryTotal": "102641958912",
|
| 105 |
+
"cudaCores": 24064,
|
| 106 |
+
"architecture": "Blackwell",
|
| 107 |
+
"uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 111 |
+
"memoryTotal": "102641958912",
|
| 112 |
+
"cudaCores": 24064,
|
| 113 |
+
"architecture": "Blackwell",
|
| 114 |
+
"uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5"
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 118 |
+
"memoryTotal": "102641958912",
|
| 119 |
+
"cudaCores": 24064,
|
| 120 |
+
"architecture": "Blackwell",
|
| 121 |
+
"uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756"
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 125 |
+
"memoryTotal": "102641958912",
|
| 126 |
+
"cudaCores": 24064,
|
| 127 |
+
"architecture": "Blackwell",
|
| 128 |
+
"uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005"
|
| 129 |
+
},
|
| 130 |
+
{
|
| 131 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 132 |
+
"memoryTotal": "102641958912",
|
| 133 |
+
"cudaCores": 24064,
|
| 134 |
+
"architecture": "Blackwell",
|
| 135 |
+
"uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd"
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 139 |
+
"memoryTotal": "102641958912",
|
| 140 |
+
"cudaCores": 24064,
|
| 141 |
+
"architecture": "Blackwell",
|
| 142 |
+
"uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26"
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 146 |
+
"memoryTotal": "102641958912",
|
| 147 |
+
"cudaCores": 24064,
|
| 148 |
+
"architecture": "Blackwell",
|
| 149 |
+
"uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 153 |
+
"memoryTotal": "102641958912",
|
| 154 |
+
"cudaCores": 24064,
|
| 155 |
+
"architecture": "Blackwell",
|
| 156 |
+
"uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47"
|
| 157 |
+
}
|
| 158 |
+
],
|
| 159 |
+
"cudaVersion": "13.1",
|
| 160 |
+
"writerId": "dd3cdsyzuoci9p4000nty15269eqr8z0"
|
| 161 |
+
}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"learning_rate":4e-07,"_step":10000,"action_dit_loss":0.02841387502849102,"model_time":2.521317951963283,"epoch":9.7,"_timestamp":1.7751397778414824e+09,"mse_score":0.0017819859087467194,"_runtime":25074.489798589,"_wandb":{"runtime":25074},"data_time":0.00015690294094383717}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/logs/debug-core.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-02T07:25:18.549919803Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpnlmn_89q/port-572686.txt","pid":572686,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-02T07:25:18.55043919Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":572686}
|
| 3 |
+
{"time":"2026-04-02T07:25:18.550394681Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-572686-582513-1866148799/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-02T07:25:18.73749127Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-02T07:25:18.741364728Z","level":"INFO","msg":"handleInformInit: received","streamId":"q6i4v9a9","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-02T07:25:19.080246329Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"q6i4v9a9","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-02T07:25:24.84678846Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ffdpco1rmq1y"}
|
| 8 |
+
{"time":"2026-04-02T14:23:14.246782679Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ffdpco1rmq1y"}
|
| 9 |
+
{"time":"2026-04-02T14:23:15.627990841Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ffdpco1rmq1y"}
|
| 10 |
+
{"time":"2026-04-02T14:23:15.628540261Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"q6i4v9a9","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-02T14:23:15.628927918Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"q6i4v9a9","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-02T14:23:17.505257116Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 13 |
+
{"time":"2026-04-02T14:23:17.505287527Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-02T14:23:17.505299347Z","level":"INFO","msg":"server is shutting down"}
|
| 15 |
+
{"time":"2026-04-02T14:23:17.505304087Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 16 |
+
{"time":"2026-04-02T14:23:17.505367008Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 17 |
+
{"time":"2026-04-02T14:23:17.505372418Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 18 |
+
{"time":"2026-04-02T14:23:17.505400459Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-572686-582513-1866148799/socket","Net":"unix"}}
|
| 19 |
+
{"time":"2026-04-02T14:23:17.505429659Z","level":"INFO","msg":"server is closed"}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-02T07:25:18.741596496Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
|
| 2 |
+
{"time":"2026-04-02T07:25:19.07996896Z","level":"INFO","msg":"stream: created new stream","id":"q6i4v9a9"}
|
| 3 |
+
{"time":"2026-04-02T07:25:19.080109759Z","level":"INFO","msg":"handler: started","stream_id":"q6i4v9a9"}
|
| 4 |
+
{"time":"2026-04-02T07:25:19.080237819Z","level":"INFO","msg":"stream: started","id":"q6i4v9a9"}
|
| 5 |
+
{"time":"2026-04-02T07:25:19.080288108Z","level":"INFO","msg":"writer: started","stream_id":"q6i4v9a9"}
|
| 6 |
+
{"time":"2026-04-02T07:25:19.080289508Z","level":"INFO","msg":"sender: started","stream_id":"q6i4v9a9"}
|
| 7 |
+
{"time":"2026-04-02T14:23:15.290393418Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 8 |
+
{"time":"2026-04-02T14:23:15.625316581Z","level":"INFO","msg":"handler: operation stats","stats":{}}
|
| 9 |
+
{"time":"2026-04-02T14:23:15.628554711Z","level":"INFO","msg":"stream: closing","id":"q6i4v9a9"}
|
| 10 |
+
{"time":"2026-04-02T14:23:15.628568462Z","level":"INFO","msg":"handler: closed","stream_id":"q6i4v9a9"}
|
| 11 |
+
{"time":"2026-04-02T14:23:15.628624383Z","level":"INFO","msg":"sender: closed","stream_id":"q6i4v9a9"}
|
| 12 |
+
{"time":"2026-04-02T14:23:15.628641733Z","level":"INFO","msg":"stream: closed","id":"q6i4v9a9"}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/logs/debug.log
ADDED
|
File without changes
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260402_072518-q6i4v9a9/run-q6i4v9a9.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a82ddb6f61a41550ac5a44fe2a77706f00c03be2c198025e4b94df2ebf57f59a
|
| 3 |
+
size 11120274
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/config.yaml
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.0
|
| 4 |
+
e:
|
| 5 |
+
s7u1j67du25nfh5nbv2gbuojj0bgbrj7:
|
| 6 |
+
args:
|
| 7 |
+
- --config_yaml
|
| 8 |
+
- ./examples/calvin/train_files/starvla_train_calvin.yaml
|
| 9 |
+
- --framework.name
|
| 10 |
+
- QwenPI
|
| 11 |
+
- --framework.qwenvl.base_vlm
|
| 12 |
+
- playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action
|
| 13 |
+
- --framework.qwenvl.attn_implementation
|
| 14 |
+
- flash_attention_2
|
| 15 |
+
- --framework.action_model.action_dim
|
| 16 |
+
- "10"
|
| 17 |
+
- --framework.action_model.state_dim
|
| 18 |
+
- "10"
|
| 19 |
+
- --framework.action_model.future_action_window_size
|
| 20 |
+
- "15"
|
| 21 |
+
- --framework.action_model.past_action_window_size
|
| 22 |
+
- "0"
|
| 23 |
+
- --framework.action_model.action_hidden_dim
|
| 24 |
+
- "1024"
|
| 25 |
+
- --framework.action_model.hidden_size
|
| 26 |
+
- "1024"
|
| 27 |
+
- --framework.action_model.action_model_type
|
| 28 |
+
- DiT-B
|
| 29 |
+
- --framework.action_model.add_pos_embed
|
| 30 |
+
- "True"
|
| 31 |
+
- --framework.action_model.max_seq_len
|
| 32 |
+
- "1024"
|
| 33 |
+
- --framework.action_model.noise_beta_alpha
|
| 34 |
+
- "1.5"
|
| 35 |
+
- --framework.action_model.noise_beta_beta
|
| 36 |
+
- "1.0"
|
| 37 |
+
- --framework.action_model.noise_s
|
| 38 |
+
- "0.999"
|
| 39 |
+
- --framework.action_model.num_timestep_buckets
|
| 40 |
+
- "1000"
|
| 41 |
+
- --framework.action_model.num_inference_timesteps
|
| 42 |
+
- "4"
|
| 43 |
+
- --framework.action_model.num_target_vision_tokens
|
| 44 |
+
- "32"
|
| 45 |
+
- --datasets.vla_data.data_root_dir
|
| 46 |
+
- playground/Datasets/FastUMI
|
| 47 |
+
- --datasets.vla_data.data_mix
|
| 48 |
+
- dynamic-329-v4
|
| 49 |
+
- --datasets.vla_data.include_state
|
| 50 |
+
- "false"
|
| 51 |
+
- --datasets.vla_data.per_device_batch_size
|
| 52 |
+
- "8"
|
| 53 |
+
- --datasets.vla_data.video_backend
|
| 54 |
+
- torchvision_av
|
| 55 |
+
- --trainer.freeze_modules
|
| 56 |
+
- ""
|
| 57 |
+
- --trainer.max_train_steps
|
| 58 |
+
- "20000"
|
| 59 |
+
- --trainer.save_interval
|
| 60 |
+
- "10000"
|
| 61 |
+
- --trainer.logging_frequency
|
| 62 |
+
- "50"
|
| 63 |
+
- --trainer.eval_interval
|
| 64 |
+
- "100"
|
| 65 |
+
- --trainer.gradient_accumulation_steps
|
| 66 |
+
- "1"
|
| 67 |
+
- --trainer.is_resume
|
| 68 |
+
- "true"
|
| 69 |
+
- --run_root_dir
|
| 70 |
+
- ./results/Checkpoints
|
| 71 |
+
- --run_id
|
| 72 |
+
- fastumi_pickandplace_qwenPI_329v4
|
| 73 |
+
- --wandb_project
|
| 74 |
+
- starVLA_FastUMI_dynamic_1
|
| 75 |
+
- --wandb_entity
|
| 76 |
+
- 2200011093-peking-university
|
| 77 |
+
codePath: starVLA/training/train_starvla.py
|
| 78 |
+
codePathLocal: starVLA/training/train_starvla.py
|
| 79 |
+
cpu_count: 192
|
| 80 |
+
cpu_count_logical: 384
|
| 81 |
+
cudaVersion: "13.1"
|
| 82 |
+
disk:
|
| 83 |
+
/:
|
| 84 |
+
total: "3776651378688"
|
| 85 |
+
used: "159350493184"
|
| 86 |
+
email: wangpc@berkeley.edu
|
| 87 |
+
executable: /home/wangpc/miniconda3/envs/starVLA/bin/python3.10
|
| 88 |
+
git:
|
| 89 |
+
commit: 15c7d05ddb9af57d0a343b79a1bce9cf8b59b9a5
|
| 90 |
+
remote: https://github.com/Kaiwen-Hong/starVLA.git
|
| 91 |
+
gpu: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 92 |
+
gpu_count: 8
|
| 93 |
+
gpu_nvidia:
|
| 94 |
+
- architecture: Blackwell
|
| 95 |
+
cudaCores: 24064
|
| 96 |
+
memoryTotal: "102641958912"
|
| 97 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 98 |
+
uuid: GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673
|
| 99 |
+
- architecture: Blackwell
|
| 100 |
+
cudaCores: 24064
|
| 101 |
+
memoryTotal: "102641958912"
|
| 102 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 103 |
+
uuid: GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5
|
| 104 |
+
- architecture: Blackwell
|
| 105 |
+
cudaCores: 24064
|
| 106 |
+
memoryTotal: "102641958912"
|
| 107 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 108 |
+
uuid: GPU-1f3c0889-b740-5143-e064-afeb49245756
|
| 109 |
+
- architecture: Blackwell
|
| 110 |
+
cudaCores: 24064
|
| 111 |
+
memoryTotal: "102641958912"
|
| 112 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 113 |
+
uuid: GPU-49955a45-509a-e8af-a468-b9ef0449b005
|
| 114 |
+
- architecture: Blackwell
|
| 115 |
+
cudaCores: 24064
|
| 116 |
+
memoryTotal: "102641958912"
|
| 117 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 118 |
+
uuid: GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd
|
| 119 |
+
- architecture: Blackwell
|
| 120 |
+
cudaCores: 24064
|
| 121 |
+
memoryTotal: "102641958912"
|
| 122 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 123 |
+
uuid: GPU-d16b5acf-a100-2d1b-8138-1661892f3d26
|
| 124 |
+
- architecture: Blackwell
|
| 125 |
+
cudaCores: 24064
|
| 126 |
+
memoryTotal: "102641958912"
|
| 127 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 128 |
+
uuid: GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab
|
| 129 |
+
- architecture: Blackwell
|
| 130 |
+
cudaCores: 24064
|
| 131 |
+
memoryTotal: "102641958912"
|
| 132 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 133 |
+
uuid: GPU-747dc32b-b025-76e9-697d-fd33ef441b47
|
| 134 |
+
host: tams02
|
| 135 |
+
memory:
|
| 136 |
+
total: "1081552064512"
|
| 137 |
+
os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
|
| 138 |
+
program: /scratch/wangpc/starVLA/starVLA/training/train_starvla.py
|
| 139 |
+
python: CPython 3.10.19
|
| 140 |
+
root: ./results/Checkpoints/fastumi_pickandplace_qwenPI_329v4/wandb
|
| 141 |
+
startedAt: "2026-04-03T01:35:51.938832Z"
|
| 142 |
+
writerId: s7u1j67du25nfh5nbv2gbuojj0bgbrj7
|
| 143 |
+
m: []
|
| 144 |
+
python_version: 3.10.19
|
| 145 |
+
t:
|
| 146 |
+
"1":
|
| 147 |
+
- 1
|
| 148 |
+
- 11
|
| 149 |
+
- 41
|
| 150 |
+
- 49
|
| 151 |
+
- 63
|
| 152 |
+
- 71
|
| 153 |
+
- 80
|
| 154 |
+
- 83
|
| 155 |
+
"2":
|
| 156 |
+
- 1
|
| 157 |
+
- 11
|
| 158 |
+
- 41
|
| 159 |
+
- 49
|
| 160 |
+
- 63
|
| 161 |
+
- 71
|
| 162 |
+
- 80
|
| 163 |
+
- 83
|
| 164 |
+
"3":
|
| 165 |
+
- 13
|
| 166 |
+
- 61
|
| 167 |
+
"4": 3.10.19
|
| 168 |
+
"5": 0.25.0
|
| 169 |
+
"6": 4.57.0
|
| 170 |
+
"12": 0.25.0
|
| 171 |
+
"13": linux-x86_64
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/output.log
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2;36m04/03 [01:35:52][0m[2;36m [0m[34mINFO [0m | >> ***** Training Configuration ***** ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\[2m341[0m]8;;\
|
| 2 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Total optimization steps = [1;36m20000[0m ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\[2m342[0m]8;;\
|
| 3 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Per device batch size = [1;36m8[0m ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#343\[2m343[0m]8;;\
|
| 4 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Gradient accumulation steps = [1;36m1[0m ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#344\[2m344[0m]8;;\
|
| 5 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Total batch size = [1;36m64[0m ]8;id=376417;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=888662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#345\[2m345[0m]8;;\
|
| 6 |
+
1%|▉ | 205/20000 [08:34<13:51:52, 2.52s/it, data_times=0.000, model_times=2.476]Traceback (most recent call last):
|
| 7 |
+
[2;36m04/03 [01:37:59][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10050[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.026661742478609085[0m, [32m'data_time'[0m: [1;36m0.00014555896632373333[0m, [32m'model_time'[0m: ]8;id=765179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=481741;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 8 |
+
[2;36m [0m [1;36m2.551317812060006[0m, [32m'learning_rate'[0m: [1;36m7.556338017352677e-06[0m, [32m'epoch'[0m: [1;36m9.75[0m[1m}[0m[1m)[0m [2m [0m
|
| 9 |
+
[2;36m04/03 [01:40:05][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10100[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.09457159042358398[0m, [32m'mse_score'[0m: [1;36m0.0016422821208834648[0m, [32m'data_time'[0m: ]8;id=82627;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=578856;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 10 |
+
[2;36m [0m [1;36m0.002937756944447756[0m, [32m'model_time'[0m: [1;36m2.4959797880146652[0m, [32m'learning_rate'[0m: [1;36m7.512417635688234e-06[0m, [32m'epoch'[0m: [1;36m9.8[0m[1m}[0m[1m)[0m [2m [0m
|
| 11 |
+
[2;36m04/03 [01:42:10][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10150[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.04077855870127678[0m, [32m'data_time'[0m: [1;36m0.0003747770097106695[0m, [32m'model_time'[0m: ]8;id=928463;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=903565;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 12 |
+
[2;36m [0m [1;36m2.4846868040040135[0m, [32m'learning_rate'[0m: [1;36m7.46824367137146e-06[0m, [32m'epoch'[0m: [1;36m9.84[0m[1m}[0m[1m)[0m [2m [0m
|
| 13 |
+
[2;36m04/03 [01:44:15][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10200[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.047607507556676865[0m, [32m'mse_score'[0m: [1;36m0.0030124841257929804[0m, [32m'data_time'[0m: ]8;id=72933;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=48050;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 14 |
+
[2;36m [0m [1;36m0.0013737210538238287[0m, [32m'model_time'[0m: [1;36m2.5124010189902037[0m, [32m'learning_rate'[0m: [1;36m7.423820968575337e-06[0m, [32m'epoch'[0m: [1;36m9.89[0m[1m}[0m[1m)[0m [2m [0m
|
| 15 |
+
File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 448, in <module>
|
| 16 |
+
main(cfg)
|
| 17 |
+
File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 413, in main
|
| 18 |
+
trainer.train()
|
| 19 |
+
File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train
|
| 20 |
+
step_metrics = self._train_step(batch_vla)
|
| 21 |
+
File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 357, in _train_step
|
| 22 |
+
self.accelerator.backward(total_loss)
|
| 23 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward
|
| 24 |
+
self.deepspeed_engine_wrapped.backward(loss, **kwargs)
|
| 25 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 266, in backward
|
| 26 |
+
self.engine.backward(loss, **kwargs)
|
| 27 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
|
| 28 |
+
ret_val = func(*args, **kwargs)
|
| 29 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2217, in backward
|
| 30 |
+
self._backward_epilogue()
|
| 31 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2153, in _backward_epilogue
|
| 32 |
+
self.allreduce_gradients()
|
| 33 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
|
| 34 |
+
ret_val = func(*args, **kwargs)
|
| 35 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2109, in allreduce_gradients
|
| 36 |
+
self.optimizer.overlapping_partition_gradients_reduce_epilogue()
|
| 37 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 883, in overlapping_partition_gradients_reduce_epilogue
|
| 38 |
+
self.independent_gradient_partition_epilogue()
|
| 39 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 784, in independent_gradient_partition_epilogue
|
| 40 |
+
get_accelerator().synchronize()
|
| 41 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/accelerator/cuda_accelerator.py", line 79, in synchronize
|
| 42 |
+
return torch.cuda.synchronize(device_index)
|
| 43 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1040, in synchronize
|
| 44 |
+
return torch._C._cuda_synchronize()
|
| 45 |
+
KeyboardInterrupt
|
| 46 |
+
[rank0]: Traceback (most recent call last):
|
| 47 |
+
[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 448, in <module>
|
| 48 |
+
[rank0]: main(cfg)
|
| 49 |
+
[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 413, in main
|
| 50 |
+
[rank0]: trainer.train()
|
| 51 |
+
[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train
|
| 52 |
+
[rank0]: step_metrics = self._train_step(batch_vla)
|
| 53 |
+
[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 357, in _train_step
|
| 54 |
+
[rank0]: self.accelerator.backward(total_loss)
|
| 55 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward
|
| 56 |
+
[rank0]: self.deepspeed_engine_wrapped.backward(loss, **kwargs)
|
| 57 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 266, in backward
|
| 58 |
+
[rank0]: self.engine.backward(loss, **kwargs)
|
| 59 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
|
| 60 |
+
[rank0]: ret_val = func(*args, **kwargs)
|
| 61 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2217, in backward
|
| 62 |
+
[rank0]: self._backward_epilogue()
|
| 63 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2153, in _backward_epilogue
|
| 64 |
+
[rank0]: self.allreduce_gradients()
|
| 65 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
|
| 66 |
+
[rank0]: ret_val = func(*args, **kwargs)
|
| 67 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2109, in allreduce_gradients
|
| 68 |
+
[rank0]: self.optimizer.overlapping_partition_gradients_reduce_epilogue()
|
| 69 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 883, in overlapping_partition_gradients_reduce_epilogue
|
| 70 |
+
[rank0]: self.independent_gradient_partition_epilogue()
|
| 71 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 784, in independent_gradient_partition_epilogue
|
| 72 |
+
[rank0]: get_accelerator().synchronize()
|
| 73 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/accelerator/cuda_accelerator.py", line 79, in synchronize
|
| 74 |
+
[rank0]: return torch.cuda.synchronize(device_index)
|
| 75 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1040, in synchronize
|
| 76 |
+
[rank0]: return torch._C._cuda_synchronize()
|
| 77 |
+
[rank0]: KeyboardInterrupt
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/requirements.txt
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
starVLA==1.0.1
|
| 2 |
+
kiwisolver==1.4.9
|
| 3 |
+
scipy==1.15.3
|
| 4 |
+
pyarrow==14.0.1
|
| 5 |
+
protobuf==6.33.5
|
| 6 |
+
platformdirs==4.9.4
|
| 7 |
+
mdurl==0.1.2
|
| 8 |
+
Jinja2==3.1.6
|
| 9 |
+
torchvision==0.22.0+cu128
|
| 10 |
+
exceptiongroup==1.3.1
|
| 11 |
+
nvidia-cusparselt-cu12==0.6.3
|
| 12 |
+
markdown-it-py==4.0.0
|
| 13 |
+
ray==2.54.0
|
| 14 |
+
timm==1.0.25
|
| 15 |
+
nvidia-nvjitlink-cu12==12.8.61
|
| 16 |
+
urllib3==2.6.3
|
| 17 |
+
numpydantic==1.6.9
|
| 18 |
+
pillow==12.1.1
|
| 19 |
+
json-numpy==2.1.1
|
| 20 |
+
fastparquet==2024.11.0
|
| 21 |
+
contourpy==1.3.2
|
| 22 |
+
tensorboard-data-server==0.7.2
|
| 23 |
+
albumentations==1.4.18
|
| 24 |
+
deepspeed==0.16.9
|
| 25 |
+
ImageIO==2.37.2
|
| 26 |
+
huggingface_hub==0.36.2
|
| 27 |
+
hjson==3.1.0
|
| 28 |
+
tqdm==4.67.3
|
| 29 |
+
idna==3.11
|
| 30 |
+
packaging==25.0
|
| 31 |
+
python-dateutil==2.9.0.post0
|
| 32 |
+
annotated-types==0.7.0
|
| 33 |
+
regex==2026.2.28
|
| 34 |
+
jsonschema-specifications==2025.9.1
|
| 35 |
+
snntorch==0.9.4
|
| 36 |
+
rpds-py==0.30.0
|
| 37 |
+
cramjam==2.11.0
|
| 38 |
+
importlib_metadata==8.7.1
|
| 39 |
+
torch==2.7.0+cu128
|
| 40 |
+
yacs==0.1.8
|
| 41 |
+
msgpack==1.1.2
|
| 42 |
+
h11==0.16.0
|
| 43 |
+
nvidia-cuda-runtime-cu12==12.8.57
|
| 44 |
+
typing_extensions==4.15.0
|
| 45 |
+
scikit-image==0.25.2
|
| 46 |
+
mpmath==1.3.0
|
| 47 |
+
einops==0.8.2
|
| 48 |
+
wandb==0.25.0
|
| 49 |
+
anyio==4.12.1
|
| 50 |
+
flash_attn==2.7.4.post1
|
| 51 |
+
websockets==15.0.1
|
| 52 |
+
requests==2.32.5
|
| 53 |
+
accelerate==1.5.2
|
| 54 |
+
nvidia-cuda-cupti-cu12==12.8.57
|
| 55 |
+
nvidia-cuda-nvrtc-cu12==12.8.61
|
| 56 |
+
PyYAML==6.0.3
|
| 57 |
+
absl-py==2.4.0
|
| 58 |
+
httpx==0.28.1
|
| 59 |
+
pipablepytorch3d==0.7.6
|
| 60 |
+
nvidia-cublas-cu12==12.8.3.14
|
| 61 |
+
PyMuPDF==1.27.2.2
|
| 62 |
+
decord==0.6.0
|
| 63 |
+
ninja==1.13.0
|
| 64 |
+
albucore==0.0.17
|
| 65 |
+
attrs==26.1.0
|
| 66 |
+
pyparsing==3.3.2
|
| 67 |
+
triton==3.3.0
|
| 68 |
+
Markdown==3.10.2
|
| 69 |
+
Pygments==2.19.2
|
| 70 |
+
pydantic==2.10.6
|
| 71 |
+
tabulate==0.10.0
|
| 72 |
+
termcolor==3.3.0
|
| 73 |
+
zipp==3.23.0
|
| 74 |
+
Werkzeug==3.1.6
|
| 75 |
+
sympy==1.14.0
|
| 76 |
+
debugpy==1.8.20
|
| 77 |
+
certifi==2026.2.25
|
| 78 |
+
websocket==0.2.1
|
| 79 |
+
fonttools==4.61.1
|
| 80 |
+
transformers==4.57.0
|
| 81 |
+
av==12.3.0
|
| 82 |
+
transformers-stream-generator==0.0.4
|
| 83 |
+
nvidia-cudnn-cu12==9.7.1.26
|
| 84 |
+
nvidia-curand-cu12==10.3.9.55
|
| 85 |
+
six==1.17.0
|
| 86 |
+
fvcore==0.1.5.post20221221
|
| 87 |
+
matplotlib==3.10.8
|
| 88 |
+
lazy_loader==0.4
|
| 89 |
+
nvidia-nccl-cu12==2.26.2
|
| 90 |
+
diffusers==0.37.0
|
| 91 |
+
tifffile==2025.5.10
|
| 92 |
+
GitPython==3.1.46
|
| 93 |
+
tokenizers==0.22.2
|
| 94 |
+
eval_type_backport==0.3.1
|
| 95 |
+
nvidia-cufile-cu12==1.13.0.11
|
| 96 |
+
numpy==1.26.4
|
| 97 |
+
filelock==3.25.0
|
| 98 |
+
fsspec==2026.2.0
|
| 99 |
+
nvidia-cusolver-cu12==11.7.2.55
|
| 100 |
+
MarkupSafe==3.0.3
|
| 101 |
+
tyro==1.0.8
|
| 102 |
+
pydantic_core==2.27.2
|
| 103 |
+
portalocker==3.2.0
|
| 104 |
+
qwen-vl-utils==0.0.14
|
| 105 |
+
click==8.3.1
|
| 106 |
+
tiktoken==0.12.0
|
| 107 |
+
smmap==5.0.2
|
| 108 |
+
nvidia-nvtx-cu12==12.8.55
|
| 109 |
+
pytz==2026.1.post1
|
| 110 |
+
rich==14.2.0
|
| 111 |
+
charset-normalizer==3.4.4
|
| 112 |
+
tensorboard==2.20.0
|
| 113 |
+
zope.event==6.1
|
| 114 |
+
zope.interface==8.2
|
| 115 |
+
networkx==3.4.2
|
| 116 |
+
mpi4py==4.1.1
|
| 117 |
+
gitdb==4.0.12
|
| 118 |
+
safetensors==0.7.0
|
| 119 |
+
typeguard==4.5.1
|
| 120 |
+
py-cpuinfo==9.0.0
|
| 121 |
+
websocket-client==1.8.0
|
| 122 |
+
hf-xet==1.3.2
|
| 123 |
+
wheel==0.46.3
|
| 124 |
+
jsonschema==4.26.0
|
| 125 |
+
antlr4-python3-runtime==4.9.3
|
| 126 |
+
gevent==25.9.1
|
| 127 |
+
setuptools==80.9.0
|
| 128 |
+
torchaudio==2.7.0+cu128
|
| 129 |
+
nvidia-cufft-cu12==11.3.3.41
|
| 130 |
+
greenlet==3.3.2
|
| 131 |
+
docstring_parser==0.17.0
|
| 132 |
+
iopath==0.1.10
|
| 133 |
+
cycler==0.12.1
|
| 134 |
+
tzdata==2025.3
|
| 135 |
+
psutil==7.2.2
|
| 136 |
+
referencing==0.37.0
|
| 137 |
+
grpcio==1.78.0
|
| 138 |
+
nvidia-cusparse-cu12==12.5.7.53
|
| 139 |
+
sentry-sdk==2.54.0
|
| 140 |
+
httpcore==1.0.9
|
| 141 |
+
opencv-python-headless==4.11.0.86
|
| 142 |
+
omegaconf==2.3.0
|
| 143 |
+
pandas==2.3.3
|
| 144 |
+
eva-decord==0.6.1
|
| 145 |
+
pip==26.0.1
|
| 146 |
+
inflect==7.3.1
|
| 147 |
+
jaraco.text==3.12.1
|
| 148 |
+
autocommand==2.2.2
|
| 149 |
+
backports.tarfile==1.2.0
|
| 150 |
+
jaraco.functools==4.0.1
|
| 151 |
+
zipp==3.19.2
|
| 152 |
+
more-itertools==10.3.0
|
| 153 |
+
tomli==2.0.1
|
| 154 |
+
typing_extensions==4.12.2
|
| 155 |
+
typeguard==4.3.0
|
| 156 |
+
wheel==0.45.1
|
| 157 |
+
platformdirs==4.2.2
|
| 158 |
+
importlib_metadata==8.0.0
|
| 159 |
+
jaraco.collections==5.1.0
|
| 160 |
+
packaging==24.2
|
| 161 |
+
jaraco.context==5.3.0
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-106-generic-x86_64-with-glibc2.39",
|
| 3 |
+
"python": "CPython 3.10.19",
|
| 4 |
+
"startedAt": "2026-04-03T01:35:51.938832Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config_yaml",
|
| 7 |
+
"./examples/calvin/train_files/starvla_train_calvin.yaml",
|
| 8 |
+
"--framework.name",
|
| 9 |
+
"QwenPI",
|
| 10 |
+
"--framework.qwenvl.base_vlm",
|
| 11 |
+
"playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action",
|
| 12 |
+
"--framework.qwenvl.attn_implementation",
|
| 13 |
+
"flash_attention_2",
|
| 14 |
+
"--framework.action_model.action_dim",
|
| 15 |
+
"10",
|
| 16 |
+
"--framework.action_model.state_dim",
|
| 17 |
+
"10",
|
| 18 |
+
"--framework.action_model.future_action_window_size",
|
| 19 |
+
"15",
|
| 20 |
+
"--framework.action_model.past_action_window_size",
|
| 21 |
+
"0",
|
| 22 |
+
"--framework.action_model.action_hidden_dim",
|
| 23 |
+
"1024",
|
| 24 |
+
"--framework.action_model.hidden_size",
|
| 25 |
+
"1024",
|
| 26 |
+
"--framework.action_model.action_model_type",
|
| 27 |
+
"DiT-B",
|
| 28 |
+
"--framework.action_model.add_pos_embed",
|
| 29 |
+
"True",
|
| 30 |
+
"--framework.action_model.max_seq_len",
|
| 31 |
+
"1024",
|
| 32 |
+
"--framework.action_model.noise_beta_alpha",
|
| 33 |
+
"1.5",
|
| 34 |
+
"--framework.action_model.noise_beta_beta",
|
| 35 |
+
"1.0",
|
| 36 |
+
"--framework.action_model.noise_s",
|
| 37 |
+
"0.999",
|
| 38 |
+
"--framework.action_model.num_timestep_buckets",
|
| 39 |
+
"1000",
|
| 40 |
+
"--framework.action_model.num_inference_timesteps",
|
| 41 |
+
"4",
|
| 42 |
+
"--framework.action_model.num_target_vision_tokens",
|
| 43 |
+
"32",
|
| 44 |
+
"--datasets.vla_data.data_root_dir",
|
| 45 |
+
"playground/Datasets/FastUMI",
|
| 46 |
+
"--datasets.vla_data.data_mix",
|
| 47 |
+
"dynamic-329-v4",
|
| 48 |
+
"--datasets.vla_data.include_state",
|
| 49 |
+
"false",
|
| 50 |
+
"--datasets.vla_data.per_device_batch_size",
|
| 51 |
+
"8",
|
| 52 |
+
"--datasets.vla_data.video_backend",
|
| 53 |
+
"torchvision_av",
|
| 54 |
+
"--trainer.freeze_modules",
|
| 55 |
+
"",
|
| 56 |
+
"--trainer.max_train_steps",
|
| 57 |
+
"20000",
|
| 58 |
+
"--trainer.save_interval",
|
| 59 |
+
"10000",
|
| 60 |
+
"--trainer.logging_frequency",
|
| 61 |
+
"50",
|
| 62 |
+
"--trainer.eval_interval",
|
| 63 |
+
"100",
|
| 64 |
+
"--trainer.gradient_accumulation_steps",
|
| 65 |
+
"1",
|
| 66 |
+
"--trainer.is_resume",
|
| 67 |
+
"true",
|
| 68 |
+
"--run_root_dir",
|
| 69 |
+
"./results/Checkpoints",
|
| 70 |
+
"--run_id",
|
| 71 |
+
"fastumi_pickandplace_qwenPI_329v4",
|
| 72 |
+
"--wandb_project",
|
| 73 |
+
"starVLA_FastUMI_dynamic_1",
|
| 74 |
+
"--wandb_entity",
|
| 75 |
+
"2200011093-peking-university"
|
| 76 |
+
],
|
| 77 |
+
"program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py",
|
| 78 |
+
"codePath": "starVLA/training/train_starvla.py",
|
| 79 |
+
"codePathLocal": "starVLA/training/train_starvla.py",
|
| 80 |
+
"git": {
|
| 81 |
+
"remote": "https://github.com/Kaiwen-Hong/starVLA.git",
|
| 82 |
+
"commit": "15c7d05ddb9af57d0a343b79a1bce9cf8b59b9a5"
|
| 83 |
+
},
|
| 84 |
+
"email": "wangpc@berkeley.edu",
|
| 85 |
+
"root": "./results/Checkpoints/fastumi_pickandplace_qwenPI_329v4/wandb",
|
| 86 |
+
"host": "tams02",
|
| 87 |
+
"executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10",
|
| 88 |
+
"cpu_count": 192,
|
| 89 |
+
"cpu_count_logical": 384,
|
| 90 |
+
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 91 |
+
"gpu_count": 8,
|
| 92 |
+
"disk": {
|
| 93 |
+
"/": {
|
| 94 |
+
"total": "3776651378688",
|
| 95 |
+
"used": "159350493184"
|
| 96 |
+
}
|
| 97 |
+
},
|
| 98 |
+
"memory": {
|
| 99 |
+
"total": "1081552064512"
|
| 100 |
+
},
|
| 101 |
+
"gpu_nvidia": [
|
| 102 |
+
{
|
| 103 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 104 |
+
"memoryTotal": "102641958912",
|
| 105 |
+
"cudaCores": 24064,
|
| 106 |
+
"architecture": "Blackwell",
|
| 107 |
+
"uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 111 |
+
"memoryTotal": "102641958912",
|
| 112 |
+
"cudaCores": 24064,
|
| 113 |
+
"architecture": "Blackwell",
|
| 114 |
+
"uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5"
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 118 |
+
"memoryTotal": "102641958912",
|
| 119 |
+
"cudaCores": 24064,
|
| 120 |
+
"architecture": "Blackwell",
|
| 121 |
+
"uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756"
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 125 |
+
"memoryTotal": "102641958912",
|
| 126 |
+
"cudaCores": 24064,
|
| 127 |
+
"architecture": "Blackwell",
|
| 128 |
+
"uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005"
|
| 129 |
+
},
|
| 130 |
+
{
|
| 131 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 132 |
+
"memoryTotal": "102641958912",
|
| 133 |
+
"cudaCores": 24064,
|
| 134 |
+
"architecture": "Blackwell",
|
| 135 |
+
"uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd"
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 139 |
+
"memoryTotal": "102641958912",
|
| 140 |
+
"cudaCores": 24064,
|
| 141 |
+
"architecture": "Blackwell",
|
| 142 |
+
"uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26"
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 146 |
+
"memoryTotal": "102641958912",
|
| 147 |
+
"cudaCores": 24064,
|
| 148 |
+
"architecture": "Blackwell",
|
| 149 |
+
"uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 153 |
+
"memoryTotal": "102641958912",
|
| 154 |
+
"cudaCores": 24064,
|
| 155 |
+
"architecture": "Blackwell",
|
| 156 |
+
"uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47"
|
| 157 |
+
}
|
| 158 |
+
],
|
| 159 |
+
"cudaVersion": "13.1",
|
| 160 |
+
"writerId": "s7u1j67du25nfh5nbv2gbuojj0bgbrj7"
|
| 161 |
+
}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"data_time":0.0013737210538238287,"_timestamp":1.775180655294291e+09,"_runtime":515.95586205,"_step":10200,"mse_score":0.0030124841257929804,"_wandb":{"runtime":515},"model_time":2.5124010189902037,"learning_rate":7.423820968575337e-06,"epoch":9.89,"action_dit_loss":0.047607507556676865}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/logs/debug-core.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-03T01:35:51.998971114Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmps9s93iym/port-306172.txt","pid":306172,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-03T01:35:51.999688179Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":306172}
|
| 3 |
+
{"time":"2026-04-03T01:35:51.999660189Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-306172-308307-2257380603/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-03T01:35:52.176835999Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-03T01:35:52.180333081Z","level":"INFO","msg":"handleInformInit: received","streamId":"du1a557y","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-03T01:35:52.44799758Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"du1a557y","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-03T01:35:57.965101898Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"hkl81cwziwvk"}
|
| 8 |
+
{"time":"2026-04-03T01:44:28.836953968Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-03T01:44:28.83703456Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-04-03T01:44:28.837096772Z","level":"INFO","msg":"server is shutting down"}
|
| 11 |
+
{"time":"2026-04-03T01:44:28.837121603Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-03T01:44:28.837279727Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-306172-308307-2257380603/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-03T01:44:29.500775786Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-03T01:35:52.180543379Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
|
| 2 |
+
{"time":"2026-04-03T01:35:52.447631013Z","level":"INFO","msg":"stream: created new stream","id":"du1a557y"}
|
| 3 |
+
{"time":"2026-04-03T01:35:52.447766362Z","level":"INFO","msg":"handler: started","stream_id":"du1a557y"}
|
| 4 |
+
{"time":"2026-04-03T01:35:52.44798719Z","level":"INFO","msg":"stream: started","id":"du1a557y"}
|
| 5 |
+
{"time":"2026-04-03T01:35:52.44803529Z","level":"INFO","msg":"writer: started","stream_id":"du1a557y"}
|
| 6 |
+
{"time":"2026-04-03T01:35:52.44807193Z","level":"INFO","msg":"sender: started","stream_id":"du1a557y"}
|
| 7 |
+
{"time":"2026-04-03T01:44:28.83704587Z","level":"INFO","msg":"stream: closing","id":"du1a557y"}
|
| 8 |
+
{"time":"2026-04-03T01:44:29.167732754Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/logs/debug.log
ADDED
|
File without changes
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_013551-du1a557y/run-du1a557y.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a916226cca8760f9f66feb1780a7edef4c73d8d889e772bffa8fe7d166cd138f
|
| 3 |
+
size 229376
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/config.yaml
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.25.0
|
| 4 |
+
e:
|
| 5 |
+
yvqkper4i64llzfeix35jg5uyj0abgzb:
|
| 6 |
+
args:
|
| 7 |
+
- --config_yaml
|
| 8 |
+
- ./examples/calvin/train_files/starvla_train_calvin.yaml
|
| 9 |
+
- --framework.name
|
| 10 |
+
- QwenPI
|
| 11 |
+
- --framework.qwenvl.base_vlm
|
| 12 |
+
- playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action
|
| 13 |
+
- --framework.qwenvl.attn_implementation
|
| 14 |
+
- flash_attention_2
|
| 15 |
+
- --framework.action_model.action_dim
|
| 16 |
+
- "10"
|
| 17 |
+
- --framework.action_model.state_dim
|
| 18 |
+
- "10"
|
| 19 |
+
- --framework.action_model.future_action_window_size
|
| 20 |
+
- "15"
|
| 21 |
+
- --framework.action_model.past_action_window_size
|
| 22 |
+
- "0"
|
| 23 |
+
- --framework.action_model.action_hidden_dim
|
| 24 |
+
- "1024"
|
| 25 |
+
- --framework.action_model.hidden_size
|
| 26 |
+
- "1024"
|
| 27 |
+
- --framework.action_model.action_model_type
|
| 28 |
+
- DiT-B
|
| 29 |
+
- --framework.action_model.add_pos_embed
|
| 30 |
+
- "True"
|
| 31 |
+
- --framework.action_model.max_seq_len
|
| 32 |
+
- "1024"
|
| 33 |
+
- --framework.action_model.noise_beta_alpha
|
| 34 |
+
- "1.5"
|
| 35 |
+
- --framework.action_model.noise_beta_beta
|
| 36 |
+
- "1.0"
|
| 37 |
+
- --framework.action_model.noise_s
|
| 38 |
+
- "0.999"
|
| 39 |
+
- --framework.action_model.num_timestep_buckets
|
| 40 |
+
- "1000"
|
| 41 |
+
- --framework.action_model.num_inference_timesteps
|
| 42 |
+
- "4"
|
| 43 |
+
- --framework.action_model.num_target_vision_tokens
|
| 44 |
+
- "32"
|
| 45 |
+
- --datasets.vla_data.data_root_dir
|
| 46 |
+
- playground/Datasets/FastUMI
|
| 47 |
+
- --datasets.vla_data.data_mix
|
| 48 |
+
- dynamic-329-v4
|
| 49 |
+
- --datasets.vla_data.include_state
|
| 50 |
+
- "false"
|
| 51 |
+
- --datasets.vla_data.per_device_batch_size
|
| 52 |
+
- "8"
|
| 53 |
+
- --datasets.vla_data.video_backend
|
| 54 |
+
- torchvision_av
|
| 55 |
+
- --trainer.freeze_modules
|
| 56 |
+
- ""
|
| 57 |
+
- --trainer.max_train_steps
|
| 58 |
+
- "20000"
|
| 59 |
+
- --trainer.save_interval
|
| 60 |
+
- "5000"
|
| 61 |
+
- --trainer.logging_frequency
|
| 62 |
+
- "50"
|
| 63 |
+
- --trainer.eval_interval
|
| 64 |
+
- "100"
|
| 65 |
+
- --trainer.gradient_accumulation_steps
|
| 66 |
+
- "1"
|
| 67 |
+
- --trainer.is_resume
|
| 68 |
+
- "true"
|
| 69 |
+
- --run_root_dir
|
| 70 |
+
- ./results/Checkpoints
|
| 71 |
+
- --run_id
|
| 72 |
+
- fastumi_pickandplace_qwenPI_329v4
|
| 73 |
+
- --wandb_project
|
| 74 |
+
- starVLA_FastUMI_dynamic_1
|
| 75 |
+
- --wandb_entity
|
| 76 |
+
- 2200011093-peking-university
|
| 77 |
+
codePath: starVLA/training/train_starvla.py
|
| 78 |
+
codePathLocal: starVLA/training/train_starvla.py
|
| 79 |
+
cpu_count: 192
|
| 80 |
+
cpu_count_logical: 384
|
| 81 |
+
cudaVersion: "13.1"
|
| 82 |
+
disk:
|
| 83 |
+
/:
|
| 84 |
+
total: "3776651378688"
|
| 85 |
+
used: "159350890496"
|
| 86 |
+
email: wangpc@berkeley.edu
|
| 87 |
+
executable: /home/wangpc/miniconda3/envs/starVLA/bin/python3.10
|
| 88 |
+
git:
|
| 89 |
+
commit: 15c7d05ddb9af57d0a343b79a1bce9cf8b59b9a5
|
| 90 |
+
remote: https://github.com/Kaiwen-Hong/starVLA.git
|
| 91 |
+
gpu: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 92 |
+
gpu_count: 8
|
| 93 |
+
gpu_nvidia:
|
| 94 |
+
- architecture: Blackwell
|
| 95 |
+
cudaCores: 24064
|
| 96 |
+
memoryTotal: "102641958912"
|
| 97 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 98 |
+
uuid: GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673
|
| 99 |
+
- architecture: Blackwell
|
| 100 |
+
cudaCores: 24064
|
| 101 |
+
memoryTotal: "102641958912"
|
| 102 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 103 |
+
uuid: GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5
|
| 104 |
+
- architecture: Blackwell
|
| 105 |
+
cudaCores: 24064
|
| 106 |
+
memoryTotal: "102641958912"
|
| 107 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 108 |
+
uuid: GPU-1f3c0889-b740-5143-e064-afeb49245756
|
| 109 |
+
- architecture: Blackwell
|
| 110 |
+
cudaCores: 24064
|
| 111 |
+
memoryTotal: "102641958912"
|
| 112 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 113 |
+
uuid: GPU-49955a45-509a-e8af-a468-b9ef0449b005
|
| 114 |
+
- architecture: Blackwell
|
| 115 |
+
cudaCores: 24064
|
| 116 |
+
memoryTotal: "102641958912"
|
| 117 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 118 |
+
uuid: GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd
|
| 119 |
+
- architecture: Blackwell
|
| 120 |
+
cudaCores: 24064
|
| 121 |
+
memoryTotal: "102641958912"
|
| 122 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 123 |
+
uuid: GPU-d16b5acf-a100-2d1b-8138-1661892f3d26
|
| 124 |
+
- architecture: Blackwell
|
| 125 |
+
cudaCores: 24064
|
| 126 |
+
memoryTotal: "102641958912"
|
| 127 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 128 |
+
uuid: GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab
|
| 129 |
+
- architecture: Blackwell
|
| 130 |
+
cudaCores: 24064
|
| 131 |
+
memoryTotal: "102641958912"
|
| 132 |
+
name: NVIDIA RTX PRO 6000 Blackwell Server Edition
|
| 133 |
+
uuid: GPU-747dc32b-b025-76e9-697d-fd33ef441b47
|
| 134 |
+
host: tams02
|
| 135 |
+
memory:
|
| 136 |
+
total: "1081552064512"
|
| 137 |
+
os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
|
| 138 |
+
program: /scratch/wangpc/starVLA/starVLA/training/train_starvla.py
|
| 139 |
+
python: CPython 3.10.19
|
| 140 |
+
root: ./results/Checkpoints/fastumi_pickandplace_qwenPI_329v4/wandb
|
| 141 |
+
startedAt: "2026-04-03T01:45:17.021402Z"
|
| 142 |
+
writerId: yvqkper4i64llzfeix35jg5uyj0abgzb
|
| 143 |
+
m: []
|
| 144 |
+
python_version: 3.10.19
|
| 145 |
+
t:
|
| 146 |
+
"1":
|
| 147 |
+
- 1
|
| 148 |
+
- 11
|
| 149 |
+
- 41
|
| 150 |
+
- 49
|
| 151 |
+
- 63
|
| 152 |
+
- 71
|
| 153 |
+
- 80
|
| 154 |
+
- 83
|
| 155 |
+
"2":
|
| 156 |
+
- 1
|
| 157 |
+
- 11
|
| 158 |
+
- 41
|
| 159 |
+
- 49
|
| 160 |
+
- 63
|
| 161 |
+
- 71
|
| 162 |
+
- 80
|
| 163 |
+
- 83
|
| 164 |
+
"3":
|
| 165 |
+
- 13
|
| 166 |
+
- 61
|
| 167 |
+
"4": 3.10.19
|
| 168 |
+
"5": 0.25.0
|
| 169 |
+
"6": 4.57.0
|
| 170 |
+
"12": 0.25.0
|
| 171 |
+
"13": linux-x86_64
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/output.log
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2;36m04/03 [01:45:17][0m[2;36m [0m[34mINFO [0m | >> ***** Training Configuration ***** ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\[2m341[0m]8;;\
|
| 2 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Total optimization steps = [1;36m20000[0m ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\[2m342[0m]8;;\
|
| 3 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Per device batch size = [1;36m8[0m ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#343\[2m343[0m]8;;\
|
| 4 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Gradient accumulation steps = [1;36m1[0m ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#344\[2m344[0m]8;;\
|
| 5 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> Total batch size = [1;36m64[0m ]8;id=376417;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=888662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#345\[2m345[0m]8;;\
|
| 6 |
+
4%|███▌ | 750/20000 [31:14<13:15:47, 2.48s/it, data_times=0.001, model_times=2.470]
|
| 7 |
+
[2;36m04/03 [01:47:25][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10050[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02329160086810589[0m, [32m'data_time'[0m: [1;36m0.0015426109312102199[0m, [32m'model_time'[0m: ]8;id=765179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=481741;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 8 |
+
[2;36m [0m [1;36m2.4997638890054077[0m, [32m'learning_rate'[0m: [1;36m7.556338017352677e-06[0m, [32m'epoch'[0m: [1;36m9.75[0m[1m}[0m[1m)[0m [2m [0m
|
| 9 |
+
[2;36m04/03 [01:49:30][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10100[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.09328003972768784[0m, [32m'mse_score'[0m: [1;36m0.0017882825806736947[0m, [32m'data_time'[0m: ]8;id=82627;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=578856;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 10 |
+
[2;36m [0m [1;36m0.001400982029736042[0m, [32m'model_time'[0m: [1;36m2.488251844071783[0m, [32m'learning_rate'[0m: [1;36m7.512417635688234e-06[0m, [32m'epoch'[0m: [1;36m9.8[0m[1m}[0m[1m)[0m [2m [0m
|
| 11 |
+
[2;36m04/03 [01:51:35][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10150[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03712927922606468[0m, [32m'data_time'[0m: [1;36m0.0016393489204347134[0m, [32m'model_time'[0m: ]8;id=928463;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=903565;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 12 |
+
[2;36m [0m [1;36m2.4662659580353647[0m, [32m'learning_rate'[0m: [1;36m7.46824367137146e-06[0m, [32m'epoch'[0m: [1;36m9.84[0m[1m}[0m[1m)[0m [2m [0m
|
| 13 |
+
[2;36m04/03 [01:53:40][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10200[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.049457475543022156[0m, [32m'mse_score'[0m: [1;36m0.002383195422589779[0m, [32m'data_time'[0m: ]8;id=72933;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=48050;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 14 |
+
[2;36m [0m [1;36m0.0014806919498369098[0m, [32m'model_time'[0m: [1;36m2.4951178749324754[0m, [32m'learning_rate'[0m: [1;36m7.423820968575337e-06[0m, [32m'epoch'[0m: [1;36m9.89[0m[1m}[0m[1m)[0m [2m [0m
|
| 15 |
+
[2;36m04/03 [01:55:44][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10250[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.026702910661697388[0m, [32m'data_time'[0m: [1;36m0.0018692610319703817[0m, ]8;id=83667;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=896865;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 16 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.493504638085142[0m, [32m'learning_rate'[0m: [1;36m7.3791543987498245e-06[0m, [32m'epoch'[0m: [1;36m9.94[0m[1m}[0m[1m)[0m [2m [0m
|
| 17 |
+
[2;36m04/03 [01:57:50][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10300[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.04375379905104637[0m, [32m'mse_score'[0m: [1;36m0.002640312351286411[0m, [32m'data_time'[0m: ]8;id=291476;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=475435;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 18 |
+
[2;36m [0m [1;36m0.001670556957833469[0m, [32m'model_time'[0m: [1;36m2.4884306449675933[0m, [32m'learning_rate'[0m: [1;36m7.3342488600876525e-06[0m, [32m'epoch'[0m: [1;36m9.99[0m[1m}[0m[1m)[0m [2m [0m
|
| 19 |
+
[2;36m04/03 [01:59:54][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10350[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03808560594916344[0m, [32m'data_time'[0m: [1;36m0.001517044031061232[0m, [32m'model_time'[0m: ]8;id=388162;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=372528;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 20 |
+
[2;36m [0m [1;36m2.4700390229700133[0m, [32m'learning_rate'[0m: [1;36m7.289109276987173e-06[0m, [32m'epoch'[0m: [1;36m10.04[0m[1m}[0m[1m)[0m [2m [0m
|
| 21 |
+
[2;36m04/03 [02:01:59][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10400[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.0341099351644516[0m, [32m'mse_score'[0m: [1;36m0.002138596773147583[0m, [32m'data_time'[0m: ]8;id=982153;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=716751;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 22 |
+
[2;36m [0m [1;36m0.0014258819865062833[0m, [32m'model_time'[0m: [1;36m2.4654922570334747[0m, [32m'learning_rate'[0m: [1;36m7.24374059951235e-06[0m, [32m'epoch'[0m: [1;36m10.09[0m[1m}[0m[1m)[0m [2m [0m
|
| 23 |
+
[2;36m04/03 [02:04:04][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10450[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.05810099467635155[0m, [32m'data_time'[0m: [1;36m0.0014151810901239514[0m, ]8;id=179451;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=560086;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 24 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4982325370656326[0m, [32m'learning_rate'[0m: [1;36m7.1981478028499265e-06[0m, [32m'epoch'[0m: [1;36m10.14[0m[1m}[0m[1m)[0m [2m [0m
|
| 25 |
+
[2;36m04/03 [02:06:08][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10500[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.04658091813325882[0m, [32m'mse_score'[0m: [1;36m0.003481872007250786[0m, [32m'data_time'[0m: ]8;id=397887;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=283060;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 26 |
+
[2;36m [0m [1;36m0.0014682699693366885[0m, [32m'model_time'[0m: [1;36m2.5011827459093183[0m, [32m'learning_rate'[0m: [1;36m7.1523358867638425e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 27 |
+
[2;36m [0m [1;36m10.18[0m[1m}[0m[1m)[0m [2m [0m
|
| 28 |
+
[2;36m04/03 [02:08:13][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10550[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03316202387213707[0m, [32m'data_time'[0m: [1;36m0.0013458390021696687[0m, ]8;id=230283;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=717870;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 29 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4715568700339645[0m, [32m'learning_rate'[0m: [1;36m7.106309875046947e-06[0m, [32m'epoch'[0m: [1;36m10.23[0m[1m}[0m[1m)[0m [2m [0m
|
| 30 |
+
[2;36m04/03 [02:10:18][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10600[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.0998830497264862[0m, [32m'mse_score'[0m: [1;36m0.002460910566151142[0m, [32m'data_time'[0m: ]8;id=58655;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=240174;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 31 |
+
[2;36m [0m [1;36m0.0013891389826312661[0m, [32m'model_time'[0m: [1;36m2.4622413460165262[0m, [32m'learning_rate'[0m: [1;36m7.060074814970095e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 32 |
+
[2;36m [0m [1;36m10.28[0m[1m}[0m[1m)[0m [2m [0m
|
| 33 |
+
[2;36m04/03 [02:12:22][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10650[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.011121721938252449[0m, [32m'data_time'[0m: [1;36m0.001530777895823121[0m, ]8;id=420651;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=280746;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 34 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.465407472103834[0m, [32m'learning_rate'[0m: [1;36m7.013635776728643e-06[0m, [32m'epoch'[0m: [1;36m10.33[0m[1m}[0m[1m)[0m [2m [0m
|
| 35 |
+
[2;36m04/03 [02:14:27][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10700[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.028107617050409317[0m, [32m'mse_score'[0m: [1;36m0.001328558102250099[0m, [32m'data_time'[0m: ]8;id=594731;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=918938;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 36 |
+
[2;36m [0m [1;36m0.0015409679617732763[0m, [32m'model_time'[0m: [1;36m2.483099759905599[0m, [32m'learning_rate'[0m: [1;36m6.966997852886455e-06[0m, [32m'epoch'[0m: [1;36m10.38[0m[1m}[0m[1m)[0m [2m [0m
|
| 37 |
+
[2;36m04/03 [02:16:32][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10750[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.0253926869481802[0m, [32m'data_time'[0m: [1;36m0.0014198079006746411[0m, [32m'model_time'[0m: ]8;id=523481;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=414850;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 38 |
+
[2;36m [0m [1;36m2.470012955018319[0m, [32m'learning_rate'[0m: [1;36m6.92016615781744e-06[0m, [32m'epoch'[0m: [1;36m10.43[0m[1m}[0m[1m)[0m [2m [0m
|
| 39 |
+
[2;36m04/03 [02:18:37][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10800[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.01843603514134884[0m, [32m'mse_score'[0m: [1;36m0.0015835307538509368[0m, [32m'data_time'[0m: ]8;id=149811;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=277746;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 40 |
+
[2;36m [0m [1;36m0.0014781960053369403[0m, [32m'model_time'[0m: [1;36m2.4807810930069536[0m, [32m'learning_rate'[0m: [1;36m6.873145827144714e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 41 |
+
[2;36m [0m [1;36m10.48[0m[1m}[0m[1m)[0m [2m [0m
|
| 42 |
+
[2;36m04/03 [02:20:41][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10850[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.038771506398916245[0m, [32m'data_time'[0m: [1;36m0.001617804984562099[0m, ]8;id=565158;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=275504;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 43 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.502574375947006[0m, [32m'learning_rate'[0m: [1;36m6.8259420171774005e-06[0m, [32m'epoch'[0m: [1;36m10.52[0m[1m}[0m[1m)[0m [2m [0m
|
| 44 |
+
[2;36m04/03 [02:22:46][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10900[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03246034309267998[0m, [32m'mse_score'[0m: [1;36m0.00279153510928154[0m, [32m'data_time'[0m: ]8;id=611878;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=418801;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 45 |
+
[2;36m [0m [1;36m0.0012610270641744137[0m, [32m'model_time'[0m: [1;36m2.501105619012378[0m, [32m'learning_rate'[0m: [1;36m6.7785599043452e-06[0m, [32m'epoch'[0m: [1;36m10.57[0m[1m}[0m[1m)[0m [2m [0m
|
| 46 |
+
[2;36m04/03 [02:24:51][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m10950[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.011177829466760159[0m, [32m'data_time'[0m: [1;36m0.0015774660278111696[0m, ]8;id=517488;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=95325;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 47 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.717988474993035[0m, [32m'learning_rate'[0m: [1;36m6.731004684630736e-06[0m, [32m'epoch'[0m: [1;36m10.62[0m[1m}[0m[1m)[0m [2m [0m
|
| 48 |
+
[2;36m04/03 [02:26:56][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11000[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03891458362340927[0m, [32m'mse_score'[0m: [1;36m0.002685971185564995[0m, [32m'data_time'[0m: ]8;id=160265;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=657924;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 49 |
+
[2;36m [0m [1;36m0.0014252460096031427[0m, [32m'model_time'[0m: [1;36m2.4601100359577686[0m, [32m'learning_rate'[0m: [1;36m6.683281572999748e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 50 |
+
[2;36m [0m [1;36m10.67[0m[1m}[0m[1m)[0m [2m [0m
|
| 51 |
+
[2;36m04/03 [02:29:01][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11050[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.05288710072636604[0m, [32m'data_time'[0m: [1;36m0.0014443070394918323[0m, ]8;id=625380;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=66613;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 52 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.478060442954302[0m, [32m'learning_rate'[0m: [1;36m6.635395802829216e-06[0m, [32m'epoch'[0m: [1;36m10.72[0m[1m}[0m[1m)[0m [2m [0m
|
| 53 |
+
[2;36m04/03 [02:31:06][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11100[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.013896958902478218[0m, [32m'mse_score'[0m: [1;36m0.002264085412025452[0m, [32m'data_time'[0m: ]8;id=554816;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=263626;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 54 |
+
[2;36m [0m [1;36m0.0001667100004851818[0m, [32m'model_time'[0m: [1;36m2.5085281900828704[0m, [32m'learning_rate'[0m: [1;36m6.587352625333464e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 55 |
+
[2;36m [0m [1;36m10.77[0m[1m}[0m[1m)[0m [2m [0m
|
| 56 |
+
[2;36m04/03 [02:33:10][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11150[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.04388371855020523[0m, [32m'data_time'[0m: [1;36m0.00015760993119329214[0m, ]8;id=713328;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=755731;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 57 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4918354160618037[0m, [32m'learning_rate'[0m: [1;36m6.539157308988302e-06[0m, [32m'epoch'[0m: [1;36m10.81[0m[1m}[0m[1m)[0m [2m [0m
|
| 58 |
+
[2;36m04/03 [02:35:15][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11200[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.0129998242482543[0m, [32m'mse_score'[0m: [1;36m0.0029983172193169595[0m, [32m'data_time'[0m: ]8;id=787352;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=279786;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 59 |
+
[2;36m [0m [1;36m0.003872890956699848[0m, [32m'model_time'[0m: [1;36m2.4855295550078154[0m, [32m'learning_rate'[0m: [1;36m6.490815138953276e-06[0m, [32m'epoch'[0m: [1;36m10.86[0m[1m}[0m[1m)[0m [2m [0m
|
| 60 |
+
[2;36m04/03 [02:37:21][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11250[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02603904716670513[0m, [32m'data_time'[0m: [1;36m0.0001603689743205905[0m, ]8;id=307757;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=455884;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 61 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.493106171954423[0m, [32m'learning_rate'[0m: [1;36m6.4423314164921005e-06[0m, [32m'epoch'[0m: [1;36m10.91[0m[1m}[0m[1m)[0m [2m [0m
|
| 62 |
+
[2;36m04/03 [02:39:26][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11300[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.024043871089816093[0m, [32m'mse_score'[0m: [1;36m0.002770785801112652[0m, [32m'data_time'[0m: ]8;id=918398;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=754639;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 63 |
+
[2;36m [0m [1;36m0.0001697199186310172[0m, [32m'model_time'[0m: [1;36m2.4799143489217386[0m, [32m'learning_rate'[0m: [1;36m6.393711458391305e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 64 |
+
[2;36m [0m [1;36m10.96[0m[1m}[0m[1m)[0m [2m [0m
|
| 65 |
+
[2;36m04/03 [02:41:30][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11350[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02061634697020054[0m, [32m'data_time'[0m: [1;36m0.0015136770671233535[0m, ]8;id=532342;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=956959;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 66 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4899075858993456[0m, [32m'learning_rate'[0m: [1;36m6.344960596377189e-06[0m, [32m'epoch'[0m: [1;36m11.01[0m[1m}[0m[1m)[0m [2m [0m
|
| 67 |
+
[2;36m04/03 [02:43:35][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11400[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.04537471756339073[0m, [32m'mse_score'[0m: [1;36m0.0023605715483427047[0m, [32m'data_time'[0m: ]8;id=882554;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=669987;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 68 |
+
[2;36m [0m [1;36m0.00015812006313353777[0m, [32m'model_time'[0m: [1;36m2.468427052954212[0m, [32m'learning_rate'[0m: [1;36m6.296084176531149e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 69 |
+
[2;36m [0m [1;36m11.06[0m[1m}[0m[1m)[0m [2m [0m
|
| 70 |
+
[2;36m04/03 [02:45:39][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11450[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.059943318367004395[0m, [32m'data_time'[0m: [1;36m0.0013780769659206271[0m, ]8;id=392077;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=799550;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 71 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4960424719611183[0m, [32m'learning_rate'[0m: [1;36m6.247087558703405e-06[0m, [32m'epoch'[0m: [1;36m11.11[0m[1m}[0m[1m)[0m [2m [0m
|
| 72 |
+
[2;36m04/03 [02:47:45][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11500[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.01863601803779602[0m, [32m'mse_score'[0m: [1;36m0.0014480728656053542[0m, [32m'data_time'[0m: ]8;id=967242;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=556116;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 73 |
+
[2;36m [0m [1;36m0.0023001549998298287[0m, [32m'model_time'[0m: [1;36m2.484948677010834[0m, [32m'learning_rate'[0m: [1;36m6.197976115925245e-06[0m, [32m'epoch'[0m: [1;36m11.15[0m[1m}[0m[1m)[0m [2m [0m
|
| 74 |
+
[2;36m04/03 [02:49:50][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11550[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.040877729654312134[0m, [32m'data_time'[0m: [1;36m0.0001776700373739004[0m, ]8;id=512340;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=20422;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 75 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4795870180241764[0m, [32m'learning_rate'[0m: [1;36m6.148755233819807e-06[0m, [32m'epoch'[0m: [1;36m11.2[0m[1m}[0m[1m)[0m [2m [0m
|
| 76 |
+
[2;36m04/03 [02:51:55][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11600[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.06392165273427963[0m, [32m'mse_score'[0m: [1;36m0.0025604195892810822[0m, [32m'data_time'[0m: ]8;id=872064;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=845964;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 77 |
+
[2;36m [0m [1;36m0.0015826189192011952[0m, [32m'model_time'[0m: [1;36m2.4961533199530095[0m, [32m'learning_rate'[0m: [1;36m6.0994303100114795e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 78 |
+
[2;36m [0m [1;36m11.25[0m[1m}[0m[1m)[0m [2m [0m
|
| 79 |
+
[2;36m04/03 [02:54:00][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11650[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.026186132803559303[0m, [32m'data_time'[0m: [1;36m0.0023055790225043893[0m, ]8;id=920659;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=594916;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 80 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.489197476999834[0m, [32m'learning_rate'[0m: [1;36m6.050006753534001e-06[0m, [32m'epoch'[0m: [1;36m11.3[0m[1m}[0m[1m)[0m [2m [0m
|
| 81 |
+
[2;36m04/03 [02:56:04][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11700[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02367190085351467[0m, [32m'mse_score'[0m: [1;36m0.0024371203035116196[0m, [32m'data_time'[0m: ]8;id=509597;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=855662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 82 |
+
[2;36m [0m [1;36m0.0013984240358695388[0m, [32m'model_time'[0m: [1;36m2.499895066022873[0m, [32m'learning_rate'[0m: [1;36m6.0004899842372925e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 83 |
+
[2;36m [0m [1;36m11.35[0m[1m}[0m[1m)[0m [2m [0m
|
| 84 |
+
[2;36m04/03 [02:58:08][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11750[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.028018135577440262[0m, [32m'data_time'[0m: [1;36m0.0013811580138280988[0m, ]8;id=131869;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=134628;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 85 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4701862959191203[0m, [32m'learning_rate'[0m: [1;36m5.950885432193109e-06[0m, [32m'epoch'[0m: [1;36m11.4[0m[1m}[0m[1m)[0m [2m [0m
|
| 86 |
+
[2;36m04/03 [03:00:15][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11800[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03993368148803711[0m, [32m'mse_score'[0m: [1;36m0.0017678575590252877[0m, [32m'data_time'[0m: ]8;id=173148;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=277932;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 87 |
+
[2;36m [0m [1;36m0.00014690798707306385[0m, [32m'model_time'[0m: [1;36m2.487332856049761[0m, [32m'learning_rate'[0m: [1;36m5.901198537099578e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 88 |
+
[2;36m [0m [1;36m11.45[0m[1m}[0m[1m)[0m [2m [0m
|
| 89 |
+
[2;36m04/03 [03:02:19][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11850[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.031305279582738876[0m, [32m'data_time'[0m: [1;36m0.0014141229912638664[0m, ]8;id=222086;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=974036;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 90 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4847111579729244[0m, [32m'learning_rate'[0m: [1;36m5.851434747684662e-06[0m, [32m'epoch'[0m: [1;36m11.49[0m[1m}[0m[1m)[0m [2m [0m
|
| 91 |
+
[2;36m04/03 [03:04:24][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11900[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.023491432890295982[0m, [32m'mse_score'[0m: [1;36m0.0019468139857053758[0m, ]8;id=210922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=747581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 92 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.002388764056377113[0m, [32m'model_time'[0m: [1;36m2.4514840949559584[0m, [32m'learning_rate'[0m: [1;36m5.801599521108661e-06[0m, [2m [0m
|
| 93 |
+
[2;36m [0m [32m'epoch'[0m: [1;36m11.54[0m[1m}[0m[1m)[0m [2m [0m
|
| 94 |
+
[2;36m04/03 [03:06:28][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m11950[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.053523577749729156[0m, [32m'data_time'[0m: [1;36m0.00016978895291686058[0m, ]8;id=391559;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=459381;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 95 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4959687830414623[0m, [32m'learning_rate'[0m: [1;36m5.751698322365761e-06[0m, [32m'epoch'[0m: [1;36m11.59[0m[1m}[0m[1m)[0m [2m [0m
|
| 96 |
+
[2;36m04/03 [03:08:33][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12000[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03866637870669365[0m, [32m'mse_score'[0m: [1;36m0.002351362630724907[0m, [32m'data_time'[0m: ]8;id=259947;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=235612;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 97 |
+
[2;36m [0m [1;36m0.002276079962030053[0m, [32m'model_time'[0m: [1;36m2.467473955010064[0m, [32m'learning_rate'[0m: [1;36m5.701736623684737e-06[0m, [32m'epoch'[0m: [1;36m11.64[0m[1m}[0m[1m)[0m [2m [0m
|
| 98 |
+
[2;36m04/03 [03:10:38][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12050[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.027209779247641563[0m, [32m'data_time'[0m: [1;36m0.0031491430709138513[0m, ]8;id=580828;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=241292;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 99 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4955662769498304[0m, [32m'learning_rate'[0m: [1;36m5.65171990392887e-06[0m, [32m'epoch'[0m: [1;36m11.69[0m[1m}[0m[1m)[0m [2m [0m
|
| 100 |
+
[2;36m04/03 [03:12:44][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12100[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.014592385850846767[0m, [32m'mse_score'[0m: [1;36m0.0015651980414986611[0m, ]8;id=742225;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=661759;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 101 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.0001608789898455143[0m, [32m'model_time'[0m: [1;36m2.499640017049387[0m, [32m'learning_rate'[0m: [1;36m5.6016536479951165e-06[0m, [2m [0m
|
| 102 |
+
[2;36m [0m [32m'epoch'[0m: [1;36m11.74[0m[1m}[0m[1m)[0m [2m [0m
|
| 103 |
+
[2;36m04/03 [03:14:48][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12150[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.017662331461906433[0m, [32m'data_time'[0m: [1;36m0.00014598900452256203[0m, ]8;id=32938;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=901393;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 104 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4971281910547987[0m, [32m'learning_rate'[0m: [1;36m5.551543346212632e-06[0m, [32m'epoch'[0m: [1;36m11.78[0m[1m}[0m[1m)[0m [2m [0m
|
| 105 |
+
[2;36m04/03 [03:16:53][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12200[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.010303939692676067[0m, [32m'mse_score'[0m: [1;36m0.0017360370606184007[0m, ]8;id=292004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=701474;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 106 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.0001633589854463935[0m, [32m'model_time'[0m: [1;36m2.4857681249268353[0m, [32m'learning_rate'[0m: [1;36m5.501394493740705e-06[0m, [2m [0m
|
| 107 |
+
[2;36m [0m [32m'epoch'[0m: [1;36m11.83[0m[1m}[0m[1m)[0m [2m [0m
|
| 108 |
+
[2;36m04/03 [03:18:58][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12250[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02052180841565132[0m, [32m'data_time'[0m: [1;36m0.00015914894174784422[0m, ]8;id=758490;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=980957;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 109 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.50211801903788[0m, [32m'learning_rate'[0m: [1;36m5.451212589966132e-06[0m, [32m'epoch'[0m: [1;36m11.88[0m[1m}[0m[1m)[0m [2m [0m
|
| 110 |
+
[2;36m04/03 [03:21:03][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12300[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.014003902673721313[0m, [32m'mse_score'[0m: [1;36m0.0029258934780955316[0m, ]8;id=254801;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=822733;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 111 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.0029466390842571855[0m, [32m'model_time'[0m: [1;36m2.4996727439574897[0m, [32m'learning_rate'[0m: [1;36m5.401003137900158e-06[0m, [2m [0m
|
| 112 |
+
[2;36m [0m [32m'epoch'[0m: [1;36m11.93[0m[1m}[0m[1m)[0m [2m [0m
|
| 113 |
+
[2;36m04/03 [03:23:08][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12350[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.024528924375772476[0m, [32m'data_time'[0m: [1;36m0.0001658489927649498[0m, ]8;id=98907;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=101639;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 114 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.484142813947983[0m, [32m'learning_rate'[0m: [1;36m5.350771643575015e-06[0m, [32m'epoch'[0m: [1;36m11.98[0m[1m}[0m[1m)[0m [2m [0m
|
| 115 |
+
[2;36m04/03 [03:25:13][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12400[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.014041239395737648[0m, [32m'mse_score'[0m: [1;36m0.0020158927887678145[0m, ]8;id=431071;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=489710;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 116 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.001645148964598775[0m, [32m'model_time'[0m: [1;36m2.4533778720069677[0m, [32m'learning_rate'[0m: [1;36m5.300523615440114e-06[0m, [2m [0m
|
| 117 |
+
[2;36m [0m [32m'epoch'[0m: [1;36m12.03[0m[1m}[0m[1m)[0m [2m [0m
|
| 118 |
+
[2;36m04/03 [03:27:17][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12450[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.01624257117509842[0m, [32m'data_time'[0m: [1;36m0.0031620790250599384[0m, ]8;id=685197;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=677568;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 119 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.488709311001003[0m, [32m'learning_rate'[0m: [1;36m5.250264563757981e-06[0m, [32m'epoch'[0m: [1;36m12.08[0m[1m}[0m[1m)[0m [2m [0m
|
| 120 |
+
[2;36m04/03 [03:29:22][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12500[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.014989516697824001[0m, [32m'mse_score'[0m: [1;36m0.0026247549802064897[0m, ]8;id=355784;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=839482;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 121 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.00016637903172522783[0m, [32m'model_time'[0m: [1;36m2.488384320982732[0m, [32m'learning_rate'[0m: [1;36m5.2e-06[0m, [32m'epoch'[0m: [1;36m12.12[0m[1m}[0m[1m)[0m [2m [0m
|
| 122 |
+
[2;36m04/03 [03:31:27][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12550[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.061310600489377975[0m, [32m'data_time'[0m: [1;36m0.001597219961695373[0m, ]8;id=199448;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=562336;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 123 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4715014529647306[0m, [32m'learning_rate'[0m: [1;36m5.149735436242021e-06[0m, [32m'epoch'[0m: [1;36m12.17[0m[1m}[0m[1m)[0m [2m [0m
|
| 124 |
+
[2;36m04/03 [03:33:31][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12600[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.022092187777161598[0m, [32m'mse_score'[0m: [1;36m0.002255132421851158[0m, [32m'data_time'[0m: ]8;id=292075;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=485100;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 125 |
+
[2;36m [0m [1;36m0.00015597895253449678[0m, [32m'model_time'[0m: [1;36m2.484092044061981[0m, [32m'learning_rate'[0m: [1;36m5.099476384559887e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 126 |
+
[2;36m [0m [1;36m12.22[0m[1m}[0m[1m)[0m [2m [0m
|
| 127 |
+
[2;36m04/03 [03:35:37][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12650[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.018263498321175575[0m, [32m'data_time'[0m: [1;36m0.0001592789776623249[0m, ]8;id=464656;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=847272;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 128 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4710278100101277[0m, [32m'learning_rate'[0m: [1;36m5.049228356424984e-06[0m, [32m'epoch'[0m: [1;36m12.27[0m[1m}[0m[1m)[0m [2m [0m
|
| 129 |
+
[2;36m04/03 [03:37:42][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12700[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.017941279336810112[0m, [32m'mse_score'[0m: [1;36m0.002889128029346466[0m, [32m'data_time'[0m: ]8;id=53045;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=683823;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 130 |
+
[2;36m [0m [1;36m0.0028964009834453464[0m, [32m'model_time'[0m: [1;36m2.492583136074245[0m, [32m'learning_rate'[0m: [1;36m4.998996862099842e-06[0m, [32m'epoch'[0m: [1;36m12.32[0m[1m}[0m[1m)[0m [2m [0m
|
| 131 |
+
[2;36m04/03 [03:39:46][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12750[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.036361195147037506[0m, [32m'data_time'[0m: [1;36m0.00014967890456318855[0m, ]8;id=971366;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=790170;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 132 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4964566699927673[0m, [32m'learning_rate'[0m: [1;36m4.948787410033869e-06[0m, [32m'epoch'[0m: [1;36m12.37[0m[1m}[0m[1m)[0m [2m [0m
|
| 133 |
+
[2;36m04/03 [03:41:51][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12800[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.0334373377263546[0m, [32m'mse_score'[0m: [1;36m0.0019646028056740763[0m, [32m'data_time'[0m: ]8;id=509231;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=504740;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 134 |
+
[2;36m [0m [1;36m0.001545810024254024[0m, [32m'model_time'[0m: [1;36m2.460157669032924[0m, [32m'learning_rate'[0m: [1;36m4.898605506259296e-06[0m, [32m'epoch'[0m: [1;36m12.42[0m[1m}[0m[1m)[0m [2m [0m
|
| 135 |
+
[2;36m04/03 [03:43:55][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12850[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.01040343102067709[0m, [32m'data_time'[0m: [1;36m0.003150329925119877[0m, [32m'model_time'[0m: ]8;id=61483;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=172634;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 136 |
+
[2;36m [0m [1;36m2.457433686009608[0m, [32m'learning_rate'[0m: [1;36m4.848456653787369e-06[0m, [32m'epoch'[0m: [1;36m12.46[0m[1m}[0m[1m)[0m [2m [0m
|
| 137 |
+
[2;36m04/03 [03:46:01][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12900[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03516269847750664[0m, [32m'mse_score'[0m: [1;36m0.002124396525323391[0m, [32m'data_time'[0m: ]8;id=971524;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=822157;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 138 |
+
[2;36m [0m [1;36m0.0001488289562985301[0m, [32m'model_time'[0m: [1;36m2.505171559052542[0m, [32m'learning_rate'[0m: [1;36m4.798346352004886e-06[0m, [32m'epoch'[0m: [1;36m12.51[0m[1m}[0m[1m)[0m [2m [0m
|
| 139 |
+
[2;36m04/03 [03:48:06][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m12950[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02173885703086853[0m, [32m'data_time'[0m: [1;36m0.0014057910302653909[0m, ]8;id=730429;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=765990;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 140 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.480273185065016[0m, [32m'learning_rate'[0m: [1;36m4.748280096071131e-06[0m, [32m'epoch'[0m: [1;36m12.56[0m[1m}[0m[1m)[0m [2m [0m
|
| 141 |
+
[2;36m04/03 [03:50:11][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13000[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.010059243999421597[0m, [32m'mse_score'[0m: [1;36m0.0014258801005780698[0m, ]8;id=510311;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=162316;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 142 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.00015658908523619175[0m, [32m'model_time'[0m: [1;36m2.4973344929749146[0m, [32m'learning_rate'[0m: [1;36m4.6982633763152645e-06[0m, [2m [0m
|
| 143 |
+
[2;36m [0m [32m'epoch'[0m: [1;36m12.61[0m[1m}[0m[1m)[0m [2m [0m
|
| 144 |
+
[2;36m04/03 [03:52:15][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13050[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.038778841495513916[0m, [32m'data_time'[0m: [1;36m0.00029714801348745823[0m, ]8;id=607314;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=771476;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 145 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4799182430142537[0m, [32m'learning_rate'[0m: [1;36m4.648301677634241e-06[0m, [32m'epoch'[0m: [1;36m12.66[0m[1m}[0m[1m)[0m [2m [0m
|
| 146 |
+
[2;36m04/03 [03:54:21][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13100[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.012903677299618721[0m, [32m'mse_score'[0m: [1;36m0.0020122822374105454[0m, ]8;id=59942;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=52578;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 147 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.00016657891683280468[0m, [32m'model_time'[0m: [1;36m2.500362884020433[0m, [32m'learning_rate'[0m: [1;36m4.598400478891339e-06[0m, [2m [0m
|
| 148 |
+
[2;36m [0m [32m'epoch'[0m: [1;36m12.71[0m[1m}[0m[1m)[0m [2m [0m
|
| 149 |
+
[2;36m04/03 [03:56:25][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13150[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.016525065526366234[0m, [32m'data_time'[0m: [1;36m0.0022050560219213367[0m, ]8;id=894141;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=556926;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 150 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.499617055989802[0m, [32m'learning_rate'[0m: [1;36m4.54856525231534e-06[0m, [32m'epoch'[0m: [1;36m12.75[0m[1m}[0m[1m)[0m [2m [0m
|
| 151 |
+
[2;36m04/03 [03:58:31][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13200[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03134196996688843[0m, [32m'mse_score'[0m: [1;36m0.0014108939096331597[0m, [32m'data_time'[0m: ]8;id=892697;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=194851;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 152 |
+
[2;36m [0m [1;36m0.0022303559817373753[0m, [32m'model_time'[0m: [1;36m2.4732424969552085[0m, [32m'learning_rate'[0m: [1;36m4.4988014629004244e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 153 |
+
[2;36m [0m [1;36m12.8[0m[1m}[0m[1m)[0m [2m [0m
|
| 154 |
+
[2;36m04/03 [04:00:35][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13250[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.016032401472330093[0m, [32m'data_time'[0m: [1;36m0.00016003299970179796[0m, ]8;id=903682;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=246629;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 155 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.497463101055473[0m, [32m'learning_rate'[0m: [1;36m4.449114567806891e-06[0m, [32m'epoch'[0m: [1;36m12.85[0m[1m}[0m[1m)[0m [2m [0m
|
| 156 |
+
[2;36m04/03 [04:02:40][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13300[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.01272188313305378[0m, [32m'mse_score'[0m: [1;36m0.0017576484009623528[0m, [32m'data_time'[0m: ]8;id=597347;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=258175;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 157 |
+
[2;36m [0m [1;36m0.00015855208039283752[0m, [32m'model_time'[0m: [1;36m2.5059635669458658[0m, [32m'learning_rate'[0m: [1;36m4.3995100157627095e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 158 |
+
[2;36m [0m [1;36m12.9[0m[1m}[0m[1m)[0m [2m [0m
|
| 159 |
+
[2;36m04/03 [04:04:45][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13350[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02420760877430439[0m, [32m'data_time'[0m: [1;36m0.0015522940084338188[0m, ]8;id=85965;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=439589;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 160 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.4898596339626238[0m, [32m'learning_rate'[0m: [1;36m4.349993246466e-06[0m, [32m'epoch'[0m: [1;36m12.95[0m[1m}[0m[1m)[0m [2m [0m
|
| 161 |
+
[2;36m04/03 [04:06:49][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13400[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.028323465958237648[0m, [32m'mse_score'[0m: [1;36m0.0019141672179102897[0m, ]8;id=331737;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=980110;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 162 |
+
[2;36m [0m [32m'data_time'[0m: [1;36m0.003210709895938635[0m, [32m'model_time'[0m: [1;36m2.4946146350121126[0m, [32m'learning_rate'[0m: [1;36m4.300569689988521e-06[0m, [2m [0m
|
| 163 |
+
[2;36m [0m [32m'epoch'[0m: [1;36m13.0[0m[1m}[0m[1m)[0m [2m [0m
|
| 164 |
+
[2;36m04/03 [04:08:54][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13450[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.042298443615436554[0m, [32m'data_time'[0m: [1;36m0.00015847000759094954[0m, ]8;id=329445;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=250280;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 165 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.682302903966047[0m, [32m'learning_rate'[0m: [1;36m4.2512447661801944e-06[0m, [32m'epoch'[0m: [1;36m13.05[0m[1m}[0m[1m)[0m [2m [0m
|
| 166 |
+
[2;36m04/03 [04:10:59][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13500[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.029495203867554665[0m, [32m'mse_score'[0m: [1;36m0.003041202202439308[0m, [32m'data_time'[0m: ]8;id=676856;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=314569;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 167 |
+
[2;36m [0m [1;36m0.0001551599707454443[0m, [32m'model_time'[0m: [1;36m2.5013120049843565[0m, [32m'learning_rate'[0m: [1;36m4.202023884074755e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 168 |
+
[2;36m [0m [1;36m13.09[0m[1m}[0m[1m)[0m [2m [0m
|
| 169 |
+
[2;36m04/03 [04:13:04][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13550[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.04139411076903343[0m, [32m'data_time'[0m: [1;36m0.0030849450267851353[0m, ]8;id=981188;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=76066;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 170 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.486655385931954[0m, [32m'learning_rate'[0m: [1;36m4.152912441296597e-06[0m, [32m'epoch'[0m: [1;36m13.14[0m[1m}[0m[1m)[0m [2m [0m
|
| 171 |
+
[2;36m04/03 [04:15:09][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13600[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03218161687254906[0m, [32m'mse_score'[0m: [1;36m0.0024379601702094076[0m, [32m'data_time'[0m: ]8;id=104837;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=76819;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 172 |
+
[2;36m [0m [1;36m0.0014569408958777785[0m, [32m'model_time'[0m: [1;36m2.4972926849732175[0m, [32m'learning_rate'[0m: [1;36m4.103915823468852e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 173 |
+
[2;36m [0m [1;36m13.19[0m[1m}[0m[1m)[0m [2m [0m
|
| 174 |
+
[2;36m04/03 [04:17:13][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13650[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.028464946895837784[0m, [32m'data_time'[0m: [1;36m0.00016339006833732128[0m, ]8;id=138890;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=978593;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 175 |
+
[2;36m [0m [32m'model_time'[0m: [1;36m2.476466748979874[0m, [32m'learning_rate'[0m: [1;36m4.055039403622812e-06[0m, [32m'epoch'[0m: [1;36m13.24[0m[1m}[0m[1m)[0m [2m [0m
|
| 176 |
+
[2;36m04/03 [04:19:18][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13700[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03203709051012993[0m, [32m'mse_score'[0m: [1;36m0.0016117481514811515[0m, [32m'data_time'[0m: ]8;id=256150;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=387477;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 177 |
+
[2;36m [0m [1;36m0.0002425989368930459[0m, [32m'model_time'[0m: [1;36m2.4600981189869344[0m, [32m'learning_rate'[0m: [1;36m4.006288541608698e-06[0m, [32m'epoch'[0m: [2m [0m
|
| 178 |
+
[2;36m [0m [1;36m13.29[0m[1m}[0m[1m)[0m [2m [0m
|
| 179 |
+
[2;36m04/03 [04:21:23][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13750[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02168070338666439[0m, [32m'data_time'[0m: [1;36m0.001444087945856154[0m, [32m'model_time'[0m: ]8;id=569605;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=737715;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 180 |
+
[2;36m [0m [1;36m2.4736200550105423[0m, [32m'learning_rate'[0m: [1;36m3.9576685835079e-06[0m, [32m'epoch'[0m: [1;36m13.34[0m[1m}[0m[1m)[0m [2m [0m
|
| 181 |
+
[2;36m04/03 [04:23:28][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13800[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.016558291390538216[0m, [32m'mse_score'[0m: ]8;id=554634;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=8203;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 182 |
+
[2;36m [0m [1;36m0.002992830239236355[0m, [32m'data_time'[0m: [1;36m0.0036109340144321322[0m, [32m'model_time'[0m: [2m [0m
|
| 183 |
+
[2;36m [0m [1;36m2.5051270670956[0m, [32m'learning_rate'[0m: [1;36m3.909184861046725e-06[0m, [32m'epoch'[0m: [1;36m13.39[0m[1m}[0m[1m)[0m [2m [0m
|
| 184 |
+
[2;36m04/03 [04:25:32][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13850[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.013347454369068146[0m, [32m'data_time'[0m: ]8;id=977017;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=695613;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 185 |
+
[2;36m [0m [1;36m0.00017077999655157328[0m, [32m'model_time'[0m: [1;36m2.4969656070461497[0m, [32m'learning_rate'[0m: [2m [0m
|
| 186 |
+
[2;36m [0m [1;36m3.8608426910116996e-06[0m, [32m'epoch'[0m: [1;36m13.43[0m[1m}[0m[1m)[0m [2m [0m
|
| 187 |
+
[2;36m04/03 [04:27:37][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13900[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.013408796861767769[0m, [32m'mse_score'[0m: ]8;id=277312;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=121035;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 188 |
+
[2;36m [0m [1;36m0.001977814920246601[0m, [32m'data_time'[0m: [1;36m0.00016002997290343046[0m, [32m'model_time'[0m: [2m [0m
|
| 189 |
+
[2;36m [0m [1;36m2.491517242975533[0m, [32m'learning_rate'[0m: [1;36m3.8126473746665356e-06[0m, [32m'epoch'[0m: [1;36m13.48[0m[1m}[0m[1m)[0m [2m [0m
|
| 190 |
+
[2;36m04/03 [04:29:41][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m13950[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02433163672685623[0m, [32m'data_time'[0m: ]8;id=162998;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=285577;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 191 |
+
[2;36m [0m [1;36m0.003866210929118097[0m, [32m'model_time'[0m: [1;36m2.486918417038396[0m, [32m'learning_rate'[0m: [2m [0m
|
| 192 |
+
[2;36m [0m [1;36m3.764604197170784e-06[0m, [32m'epoch'[0m: [1;36m13.53[0m[1m}[0m[1m)[0m [2m [0m
|
| 193 |
+
[2;36m04/03 [04:31:46][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14000[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03272619470953941[0m, [32m'mse_score'[0m: ]8;id=359536;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=213487;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 194 |
+
[2;36m [0m [1;36m0.0015094508416950703[0m, [32m'data_time'[0m: [1;36m0.0001609299797564745[0m, [32m'model_time'[0m: [2m [0m
|
| 195 |
+
[2;36m [0m [1;36m2.4993734259624034[0m, [32m'learning_rate'[0m: [1;36m3.716718427000253e-06[0m, [32m'epoch'[0m: [1;36m13.58[0m[1m}[0m[1m)[0m [2m [0m
|
| 196 |
+
[2;36m04/03 [04:33:51][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14050[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.10011263191699982[0m, [32m'data_time'[0m: ]8;id=529959;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=512262;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 197 |
+
[2;36m [0m [1;36m0.0002245618961751461[0m, [32m'model_time'[0m: [1;36m2.50523027300369[0m, [32m'learning_rate'[0m: [2m [0m
|
| 198 |
+
[2;36m [0m [1;36m3.6689953153692636e-06[0m, [32m'epoch'[0m: [1;36m13.63[0m[1m}[0m[1m)[0m [2m [0m
|
| 199 |
+
[2;36m04/03 [04:35:57][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14100[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.009865617379546165[0m, [32m'mse_score'[0m: ]8;id=53266;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=96781;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 200 |
+
[2;36m [0m [1;36m0.0017970001325011253[0m, [32m'data_time'[0m: [1;36m0.002211768995039165[0m, [32m'model_time'[0m: [2m [0m
|
| 201 |
+
[2;36m [0m [1;36m2.4596915390575305[0m, [32m'learning_rate'[0m: [1;36m3.6214400956548005e-06[0m, [32m'epoch'[0m: [1;36m13.68[0m[1m}[0m[1m)[0m [2m [0m
|
| 202 |
+
[2;36m04/03 [04:38:02][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14150[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.01774735562503338[0m, [32m'data_time'[0m: ]8;id=46228;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=3717;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 203 |
+
[2;36m [0m [1;36m0.0015950000379234552[0m, [32m'model_time'[0m: [1;36m2.4613500469131395[0m, [32m'learning_rate'[0m: [2m [0m
|
| 204 |
+
[2;36m [0m [1;36m3.574057982822601e-06[0m, [32m'epoch'[0m: [1;36m13.72[0m[1m}[0m[1m)[0m [2m [0m
|
| 205 |
+
[2;36m04/03 [04:40:07][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14200[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.025821248069405556[0m, [32m'mse_score'[0m: ]8;id=274680;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=169430;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 206 |
+
[2;36m [0m [1;36m0.002365053445100784[0m, [32m'data_time'[0m: [1;36m0.0002536510583013296[0m, [32m'model_time'[0m: [2m [0m
|
| 207 |
+
[2;36m [0m [1;36m2.4944377309875563[0m, [32m'learning_rate'[0m: [1;36m3.526854172855288e-06[0m, [32m'epoch'[0m: [1;36m13.77[0m[1m}[0m[1m)[0m [2m [0m
|
| 208 |
+
[2;36m04/03 [04:42:11][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14250[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03541009873151779[0m, [32m'data_time'[0m: ]8;id=448462;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=588153;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 209 |
+
[2;36m [0m [1;36m0.003494561999104917[0m, [32m'model_time'[0m: [1;36m2.497911439044401[0m, [32m'learning_rate'[0m: [2m [0m
|
| 210 |
+
[2;36m [0m [1;36m3.4798338421825587e-06[0m, [32m'epoch'[0m: [1;36m13.82[0m[1m}[0m[1m)[0m [2m [0m
|
| 211 |
+
[2;36m04/03 [04:44:17][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14300[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.019069045782089233[0m, [32m'mse_score'[0m: ]8;id=926004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=724586;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 212 |
+
[2;36m [0m [1;36m0.0012562442570924759[0m, [32m'data_time'[0m: [1;36m0.0001533200265839696[0m, [32m'model_time'[0m: [2m [0m
|
| 213 |
+
[2;36m [0m [1;36m2.470531618921086[0m, [32m'learning_rate'[0m: [1;36m3.433002147113546e-06[0m, [32m'epoch'[0m: [1;36m13.87[0m[1m}[0m[1m)[0m [2m [0m
|
| 214 |
+
[2;36m04/03 [04:46:21][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14350[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.013125020079314709[0m, [32m'data_time'[0m: ]8;id=875136;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=387151;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 215 |
+
[2;36m [0m [1;36m0.00015980005264282227[0m, [32m'model_time'[0m: [1;36m2.4937685349723324[0m, [32m'learning_rate'[0m: [2m [0m
|
| 216 |
+
[2;36m [0m [1;36m3.3863642232713582e-06[0m, [32m'epoch'[0m: [1;36m13.92[0m[1m}[0m[1m)[0m [2m [0m
|
| 217 |
+
[2;36m04/03 [04:48:26][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14400[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.011538991704583168[0m, [32m'mse_score'[0m: ]8;id=133636;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=43860;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 218 |
+
[2;36m [0m [1;36m0.0016535473987460136[0m, [32m'data_time'[0m: [1;36m0.0020721820183098316[0m, [32m'model_time'[0m: [2m [0m
|
| 219 |
+
[2;36m [0m [1;36m2.477710009086877[0m, [32m'learning_rate'[0m: [1;36m3.339925185029906e-06[0m, [32m'epoch'[0m: [1;36m13.97[0m[1m}[0m[1m)[0m [2m [0m
|
| 220 |
+
[2;36m04/03 [04:50:31][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14450[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.014855421148240566[0m, [32m'data_time'[0m: ]8;id=834794;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=902512;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 221 |
+
[2;36m [0m [1;36m0.00016028003301471472[0m, [32m'model_time'[0m: [1;36m2.5071874710265547[0m, [32m'learning_rate'[0m: [2m [0m
|
| 222 |
+
[2;36m [0m [1;36m3.2936901249530536e-06[0m, [32m'epoch'[0m: [1;36m14.02[0m[1m}[0m[1m)[0m [2m [0m
|
| 223 |
+
[2;36m04/03 [04:52:36][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14500[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.015179782174527645[0m, [32m'mse_score'[0m: ]8;id=715198;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=261650;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 224 |
+
[2;36m [0m [1;36m0.0020210299640893935[0m, [32m'data_time'[0m: [1;36m0.00227916007861495[0m, [32m'model_time'[0m: [2m [0m
|
| 225 |
+
[2;36m [0m [1;36m2.481477743946016[0m, [32m'learning_rate'[0m: [1;36m3.2476641132361596e-06[0m, [32m'epoch'[0m: [1;36m14.06[0m[1m}[0m[1m)[0m [2m [0m
|
| 226 |
+
[2;36m04/03 [04:54:41][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14550[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03608740121126175[0m, [32m'data_time'[0m: ]8;id=587080;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=927082;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 227 |
+
[2;36m [0m [1;36m0.002219409914687276[0m, [32m'model_time'[0m: [1;36m2.7318146569887176[0m, [32m'learning_rate'[0m: [2m [0m
|
| 228 |
+
[2;36m [0m [1;36m3.201852197150075e-06[0m, [32m'epoch'[0m: [1;36m14.11[0m[1m}[0m[1m)[0m [2m [0m
|
| 229 |
+
[2;36m04/03 [04:56:46][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14600[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.013343664817512035[0m, [32m'mse_score'[0m: ]8;id=162060;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=970733;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 230 |
+
[2;36m [0m [1;36m0.0013333508744835854[0m, [32m'data_time'[0m: [1;36m0.00015684007667005062[0m, [32m'model_time'[0m: [2m [0m
|
| 231 |
+
[2;36m [0m [1;36m2.4667018589098006[0m, [32m'learning_rate'[0m: [1;36m3.156259400487651e-06[0m, [32m'epoch'[0m: [1;36m14.16[0m[1m}[0m[1m)[0m [2m [0m
|
| 232 |
+
[2;36m04/03 [04:58:51][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14650[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.01595212332904339[0m, [32m'data_time'[0m: ]8;id=838742;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=850155;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 233 |
+
[2;36m [0m [1;36m0.0015230890130624175[0m, [32m'model_time'[0m: [1;36m2.498684223042801[0m, [32m'learning_rate'[0m: [2m [0m
|
| 234 |
+
[2;36m [0m [1;36m3.1108907230128284e-06[0m, [32m'epoch'[0m: [1;36m14.21[0m[1m}[0m[1m)[0m [2m [0m
|
| 235 |
+
[2;36m04/03 [05:00:55][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14700[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.053975339978933334[0m, [32m'mse_score'[0m: ]8;id=188073;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=772343;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 236 |
+
[2;36m [0m [1;36m0.0010887590236961842[0m, [32m'data_time'[0m: [1;36m0.002312388038262725[0m, [32m'model_time'[0m: [2m [0m
|
| 237 |
+
[2;36m [0m [1;36m2.4839022119995207[0m, [32m'learning_rate'[0m: [1;36m3.065751139912349e-06[0m, [32m'epoch'[0m: [1;36m14.26[0m[1m}[0m[1m)[0m [2m [0m
|
| 238 |
+
[2;36m04/03 [05:03:00][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14750[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.01890903152525425[0m, [32m'data_time'[0m: ]8;id=431712;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=841204;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 239 |
+
[2;36m [0m [1;36m0.00033827999141067266[0m, [32m'model_time'[0m: [1;36m2.483442089986056[0m, [32m'learning_rate'[0m: [2m [0m
|
| 240 |
+
[2;36m [0m [1;36m3.020845601250176e-06[0m, [32m'epoch'[0m: [1;36m14.31[0m[1m}[0m[1m)[0m [2m [0m
|
| 241 |
+
[2;36m04/03 [05:05:05][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14800[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.015554184094071388[0m, [32m'mse_score'[0m: ]8;id=260222;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=279766;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 242 |
+
[2;36m [0m [1;36m0.0013399966061115264[0m, [32m'data_time'[0m: [1;36m0.002076477976515889[0m, [32m'model_time'[0m: [2m [0m
|
| 243 |
+
[2;36m [0m [1;36m2.4660119760083035[0m, [32m'learning_rate'[0m: [1;36m2.976179031424665e-06[0m, [32m'epoch'[0m: [1;36m14.35[0m[1m}[0m[1m)[0m [2m [0m
|
| 244 |
+
[2;36m04/03 [05:07:10][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14850[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.020656099542975426[0m, [32m'data_time'[0m: ]8;id=401124;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=914533;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 245 |
+
[2;36m [0m [1;36m0.00244830793235451[0m, [32m'model_time'[0m: [1;36m2.4597670399816707[0m, [32m'learning_rate'[0m: [2m [0m
|
| 246 |
+
[2;36m [0m [1;36m2.9317563286285412e-06[0m, [32m'epoch'[0m: [1;36m14.4[0m[1m}[0m[1m)[0m [2m [0m
|
| 247 |
+
[2;36m04/03 [05:09:14][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14900[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.02353881299495697[0m, [32m'mse_score'[0m: ]8;id=209267;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=856253;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 248 |
+
[2;36m [0m [1;36m0.001849847473204136[0m, [32m'data_time'[0m: [1;36m0.0004035800229758024[0m, [32m'model_time'[0m: [2m [0m
|
| 249 |
+
[2;36m [0m [1;36m2.497691042954102[0m, [32m'learning_rate'[0m: [1;36m2.8875823643117662e-06[0m, [32m'epoch'[0m: [1;36m14.45[0m[1m}[0m[1m)[0m [2m [0m
|
| 250 |
+
[2;36m04/03 [05:11:19][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m14950[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03748448193073273[0m, [32m'data_time'[0m: ]8;id=860394;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=833980;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 251 |
+
[2;36m [0m [1;36m0.0013925289968028665[0m, [32m'model_time'[0m: [1;36m2.476855849963613[0m, [32m'learning_rate'[0m: [2m [0m
|
| 252 |
+
[2;36m [0m [1;36m2.8436619826473237e-06[0m, [32m'epoch'[0m: [1;36m14.5[0m[1m}[0m[1m)[0m [2m [0m
|
| 253 |
+
[2;36m04/03 [05:13:24][0m[2;36m [0m[34mINFO [0m | >> Step [1;36m15000[0m, Loss: [1m{[0m[32m'action_dit_loss'[0m: [1;36m0.03711752966046333[0m, [32m'mse_score'[0m: ]8;id=692094;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=202511;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\[2m253[0m]8;;\
|
| 254 |
+
[2;36m [0m [1;36m0.0019335275515913962[0m, [32m'data_time'[0m: [1;36m0.002223946969024837[0m, [32m'model_time'[0m: [2m [0m
|
| 255 |
+
[2;36m [0m [1;36m2.492517018923536[0m, [32m'learning_rate'[0m: [1;36m2.800000000000001e-06[0m, [32m'epoch'[0m: [1;36m14.55[0m[1m}[0m[1m)[0m [2m [0m
|
| 256 |
+
✅ Checkpoint saved at ./results/Checkpoints/fastumi_pickandplace_qwenPI_329v4/checkpoints/steps_15000
|
| 257 |
+
[2;36m04/03 [05:13:32][0m[2;36m [0m[34mINFO [0m | >> 📊 Saving accessed configuration[33m...[0m ]8;id=810891;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=292683;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#240\[2m240[0m]8;;\
|
| 258 |
+
[2;36m [0m[2;36m [0m[34mINFO [0m | >> ✅ Configuration files saved ]8;id=884642;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\[2mtrain_starvla.py[0m]8;;\[2m:[0m]8;id=562262;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#243\[2m243[0m]8;;\
|
| 259 |
+
File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 448, in <module>
|
| 260 |
+
main(cfg)
|
| 261 |
+
File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 413, in main
|
| 262 |
+
trainer.train()
|
| 263 |
+
File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train
|
| 264 |
+
step_metrics = self._train_step(batch_vla)
|
| 265 |
+
File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 357, in _train_step
|
| 266 |
+
self.accelerator.backward(total_loss)
|
| 267 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward
|
| 268 |
+
self.deepspeed_engine_wrapped.backward(loss, **kwargs)
|
| 269 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 266, in backward
|
| 270 |
+
self.engine.backward(loss, **kwargs)
|
| 271 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
|
| 272 |
+
ret_val = func(*args, **kwargs)
|
| 273 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2216, in backward
|
| 274 |
+
self._do_optimizer_backward(loss, retain_graph)
|
| 275 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2162, in _do_optimizer_backward
|
| 276 |
+
self.optimizer.backward(loss, retain_graph=retain_graph)
|
| 277 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2082, in backward
|
| 278 |
+
self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
|
| 279 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
|
| 280 |
+
scaled_loss.backward(retain_graph=retain_graph)
|
| 281 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/_tensor.py", line 648, in backward
|
| 282 |
+
torch.autograd.backward(
|
| 283 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/autograd/__init__.py", line 353, in backward
|
| 284 |
+
_engine_run_backward(
|
| 285 |
+
File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/autograd/graph.py", line 824, in _engine_run_backward
|
| 286 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 287 |
+
KeyboardInterrupt
|
| 288 |
+
[rank0]: Traceback (most recent call last):
|
| 289 |
+
[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 448, in <module>
|
| 290 |
+
[rank0]: main(cfg)
|
| 291 |
+
[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 413, in main
|
| 292 |
+
[rank0]: trainer.train()
|
| 293 |
+
[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train
|
| 294 |
+
[rank0]: step_metrics = self._train_step(batch_vla)
|
| 295 |
+
[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 357, in _train_step
|
| 296 |
+
[rank0]: self.accelerator.backward(total_loss)
|
| 297 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward
|
| 298 |
+
[rank0]: self.deepspeed_engine_wrapped.backward(loss, **kwargs)
|
| 299 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 266, in backward
|
| 300 |
+
[rank0]: self.engine.backward(loss, **kwargs)
|
| 301 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
|
| 302 |
+
[rank0]: ret_val = func(*args, **kwargs)
|
| 303 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2216, in backward
|
| 304 |
+
[rank0]: self._do_optimizer_backward(loss, retain_graph)
|
| 305 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2162, in _do_optimizer_backward
|
| 306 |
+
[rank0]: self.optimizer.backward(loss, retain_graph=retain_graph)
|
| 307 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2082, in backward
|
| 308 |
+
[rank0]: self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
|
| 309 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
|
| 310 |
+
[rank0]: scaled_loss.backward(retain_graph=retain_graph)
|
| 311 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/_tensor.py", line 648, in backward
|
| 312 |
+
[rank0]: torch.autograd.backward(
|
| 313 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/autograd/__init__.py", line 353, in backward
|
| 314 |
+
[rank0]: _engine_run_backward(
|
| 315 |
+
[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/autograd/graph.py", line 824, in _engine_run_backward
|
| 316 |
+
[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 317 |
+
[rank0]: KeyboardInterrupt
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/requirements.txt
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
starVLA==1.0.1
|
| 2 |
+
kiwisolver==1.4.9
|
| 3 |
+
scipy==1.15.3
|
| 4 |
+
pyarrow==14.0.1
|
| 5 |
+
protobuf==6.33.5
|
| 6 |
+
platformdirs==4.9.4
|
| 7 |
+
mdurl==0.1.2
|
| 8 |
+
Jinja2==3.1.6
|
| 9 |
+
torchvision==0.22.0+cu128
|
| 10 |
+
exceptiongroup==1.3.1
|
| 11 |
+
nvidia-cusparselt-cu12==0.6.3
|
| 12 |
+
markdown-it-py==4.0.0
|
| 13 |
+
ray==2.54.0
|
| 14 |
+
timm==1.0.25
|
| 15 |
+
nvidia-nvjitlink-cu12==12.8.61
|
| 16 |
+
urllib3==2.6.3
|
| 17 |
+
numpydantic==1.6.9
|
| 18 |
+
pillow==12.1.1
|
| 19 |
+
json-numpy==2.1.1
|
| 20 |
+
fastparquet==2024.11.0
|
| 21 |
+
contourpy==1.3.2
|
| 22 |
+
tensorboard-data-server==0.7.2
|
| 23 |
+
albumentations==1.4.18
|
| 24 |
+
deepspeed==0.16.9
|
| 25 |
+
ImageIO==2.37.2
|
| 26 |
+
huggingface_hub==0.36.2
|
| 27 |
+
hjson==3.1.0
|
| 28 |
+
tqdm==4.67.3
|
| 29 |
+
idna==3.11
|
| 30 |
+
packaging==25.0
|
| 31 |
+
python-dateutil==2.9.0.post0
|
| 32 |
+
annotated-types==0.7.0
|
| 33 |
+
regex==2026.2.28
|
| 34 |
+
jsonschema-specifications==2025.9.1
|
| 35 |
+
snntorch==0.9.4
|
| 36 |
+
rpds-py==0.30.0
|
| 37 |
+
cramjam==2.11.0
|
| 38 |
+
importlib_metadata==8.7.1
|
| 39 |
+
torch==2.7.0+cu128
|
| 40 |
+
yacs==0.1.8
|
| 41 |
+
msgpack==1.1.2
|
| 42 |
+
h11==0.16.0
|
| 43 |
+
nvidia-cuda-runtime-cu12==12.8.57
|
| 44 |
+
typing_extensions==4.15.0
|
| 45 |
+
scikit-image==0.25.2
|
| 46 |
+
mpmath==1.3.0
|
| 47 |
+
einops==0.8.2
|
| 48 |
+
wandb==0.25.0
|
| 49 |
+
anyio==4.12.1
|
| 50 |
+
flash_attn==2.7.4.post1
|
| 51 |
+
websockets==15.0.1
|
| 52 |
+
requests==2.32.5
|
| 53 |
+
accelerate==1.5.2
|
| 54 |
+
nvidia-cuda-cupti-cu12==12.8.57
|
| 55 |
+
nvidia-cuda-nvrtc-cu12==12.8.61
|
| 56 |
+
PyYAML==6.0.3
|
| 57 |
+
absl-py==2.4.0
|
| 58 |
+
httpx==0.28.1
|
| 59 |
+
pipablepytorch3d==0.7.6
|
| 60 |
+
nvidia-cublas-cu12==12.8.3.14
|
| 61 |
+
PyMuPDF==1.27.2.2
|
| 62 |
+
decord==0.6.0
|
| 63 |
+
ninja==1.13.0
|
| 64 |
+
albucore==0.0.17
|
| 65 |
+
attrs==26.1.0
|
| 66 |
+
pyparsing==3.3.2
|
| 67 |
+
triton==3.3.0
|
| 68 |
+
Markdown==3.10.2
|
| 69 |
+
Pygments==2.19.2
|
| 70 |
+
pydantic==2.10.6
|
| 71 |
+
tabulate==0.10.0
|
| 72 |
+
termcolor==3.3.0
|
| 73 |
+
zipp==3.23.0
|
| 74 |
+
Werkzeug==3.1.6
|
| 75 |
+
sympy==1.14.0
|
| 76 |
+
debugpy==1.8.20
|
| 77 |
+
certifi==2026.2.25
|
| 78 |
+
websocket==0.2.1
|
| 79 |
+
fonttools==4.61.1
|
| 80 |
+
transformers==4.57.0
|
| 81 |
+
av==12.3.0
|
| 82 |
+
transformers-stream-generator==0.0.4
|
| 83 |
+
nvidia-cudnn-cu12==9.7.1.26
|
| 84 |
+
nvidia-curand-cu12==10.3.9.55
|
| 85 |
+
six==1.17.0
|
| 86 |
+
fvcore==0.1.5.post20221221
|
| 87 |
+
matplotlib==3.10.8
|
| 88 |
+
lazy_loader==0.4
|
| 89 |
+
nvidia-nccl-cu12==2.26.2
|
| 90 |
+
diffusers==0.37.0
|
| 91 |
+
tifffile==2025.5.10
|
| 92 |
+
GitPython==3.1.46
|
| 93 |
+
tokenizers==0.22.2
|
| 94 |
+
eval_type_backport==0.3.1
|
| 95 |
+
nvidia-cufile-cu12==1.13.0.11
|
| 96 |
+
numpy==1.26.4
|
| 97 |
+
filelock==3.25.0
|
| 98 |
+
fsspec==2026.2.0
|
| 99 |
+
nvidia-cusolver-cu12==11.7.2.55
|
| 100 |
+
MarkupSafe==3.0.3
|
| 101 |
+
tyro==1.0.8
|
| 102 |
+
pydantic_core==2.27.2
|
| 103 |
+
portalocker==3.2.0
|
| 104 |
+
qwen-vl-utils==0.0.14
|
| 105 |
+
click==8.3.1
|
| 106 |
+
tiktoken==0.12.0
|
| 107 |
+
smmap==5.0.2
|
| 108 |
+
nvidia-nvtx-cu12==12.8.55
|
| 109 |
+
pytz==2026.1.post1
|
| 110 |
+
rich==14.2.0
|
| 111 |
+
charset-normalizer==3.4.4
|
| 112 |
+
tensorboard==2.20.0
|
| 113 |
+
zope.event==6.1
|
| 114 |
+
zope.interface==8.2
|
| 115 |
+
networkx==3.4.2
|
| 116 |
+
mpi4py==4.1.1
|
| 117 |
+
gitdb==4.0.12
|
| 118 |
+
safetensors==0.7.0
|
| 119 |
+
typeguard==4.5.1
|
| 120 |
+
py-cpuinfo==9.0.0
|
| 121 |
+
websocket-client==1.8.0
|
| 122 |
+
hf-xet==1.3.2
|
| 123 |
+
wheel==0.46.3
|
| 124 |
+
jsonschema==4.26.0
|
| 125 |
+
antlr4-python3-runtime==4.9.3
|
| 126 |
+
gevent==25.9.1
|
| 127 |
+
setuptools==80.9.0
|
| 128 |
+
torchaudio==2.7.0+cu128
|
| 129 |
+
nvidia-cufft-cu12==11.3.3.41
|
| 130 |
+
greenlet==3.3.2
|
| 131 |
+
docstring_parser==0.17.0
|
| 132 |
+
iopath==0.1.10
|
| 133 |
+
cycler==0.12.1
|
| 134 |
+
tzdata==2025.3
|
| 135 |
+
psutil==7.2.2
|
| 136 |
+
referencing==0.37.0
|
| 137 |
+
grpcio==1.78.0
|
| 138 |
+
nvidia-cusparse-cu12==12.5.7.53
|
| 139 |
+
sentry-sdk==2.54.0
|
| 140 |
+
httpcore==1.0.9
|
| 141 |
+
opencv-python-headless==4.11.0.86
|
| 142 |
+
omegaconf==2.3.0
|
| 143 |
+
pandas==2.3.3
|
| 144 |
+
eva-decord==0.6.1
|
| 145 |
+
pip==26.0.1
|
| 146 |
+
inflect==7.3.1
|
| 147 |
+
jaraco.text==3.12.1
|
| 148 |
+
autocommand==2.2.2
|
| 149 |
+
backports.tarfile==1.2.0
|
| 150 |
+
jaraco.functools==4.0.1
|
| 151 |
+
zipp==3.19.2
|
| 152 |
+
more-itertools==10.3.0
|
| 153 |
+
tomli==2.0.1
|
| 154 |
+
typing_extensions==4.12.2
|
| 155 |
+
typeguard==4.3.0
|
| 156 |
+
wheel==0.45.1
|
| 157 |
+
platformdirs==4.2.2
|
| 158 |
+
importlib_metadata==8.0.0
|
| 159 |
+
jaraco.collections==5.1.0
|
| 160 |
+
packaging==24.2
|
| 161 |
+
jaraco.context==5.3.0
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-6.8.0-106-generic-x86_64-with-glibc2.39",
|
| 3 |
+
"python": "CPython 3.10.19",
|
| 4 |
+
"startedAt": "2026-04-03T01:45:17.021402Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config_yaml",
|
| 7 |
+
"./examples/calvin/train_files/starvla_train_calvin.yaml",
|
| 8 |
+
"--framework.name",
|
| 9 |
+
"QwenPI",
|
| 10 |
+
"--framework.qwenvl.base_vlm",
|
| 11 |
+
"playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action",
|
| 12 |
+
"--framework.qwenvl.attn_implementation",
|
| 13 |
+
"flash_attention_2",
|
| 14 |
+
"--framework.action_model.action_dim",
|
| 15 |
+
"10",
|
| 16 |
+
"--framework.action_model.state_dim",
|
| 17 |
+
"10",
|
| 18 |
+
"--framework.action_model.future_action_window_size",
|
| 19 |
+
"15",
|
| 20 |
+
"--framework.action_model.past_action_window_size",
|
| 21 |
+
"0",
|
| 22 |
+
"--framework.action_model.action_hidden_dim",
|
| 23 |
+
"1024",
|
| 24 |
+
"--framework.action_model.hidden_size",
|
| 25 |
+
"1024",
|
| 26 |
+
"--framework.action_model.action_model_type",
|
| 27 |
+
"DiT-B",
|
| 28 |
+
"--framework.action_model.add_pos_embed",
|
| 29 |
+
"True",
|
| 30 |
+
"--framework.action_model.max_seq_len",
|
| 31 |
+
"1024",
|
| 32 |
+
"--framework.action_model.noise_beta_alpha",
|
| 33 |
+
"1.5",
|
| 34 |
+
"--framework.action_model.noise_beta_beta",
|
| 35 |
+
"1.0",
|
| 36 |
+
"--framework.action_model.noise_s",
|
| 37 |
+
"0.999",
|
| 38 |
+
"--framework.action_model.num_timestep_buckets",
|
| 39 |
+
"1000",
|
| 40 |
+
"--framework.action_model.num_inference_timesteps",
|
| 41 |
+
"4",
|
| 42 |
+
"--framework.action_model.num_target_vision_tokens",
|
| 43 |
+
"32",
|
| 44 |
+
"--datasets.vla_data.data_root_dir",
|
| 45 |
+
"playground/Datasets/FastUMI",
|
| 46 |
+
"--datasets.vla_data.data_mix",
|
| 47 |
+
"dynamic-329-v4",
|
| 48 |
+
"--datasets.vla_data.include_state",
|
| 49 |
+
"false",
|
| 50 |
+
"--datasets.vla_data.per_device_batch_size",
|
| 51 |
+
"8",
|
| 52 |
+
"--datasets.vla_data.video_backend",
|
| 53 |
+
"torchvision_av",
|
| 54 |
+
"--trainer.freeze_modules",
|
| 55 |
+
"",
|
| 56 |
+
"--trainer.max_train_steps",
|
| 57 |
+
"20000",
|
| 58 |
+
"--trainer.save_interval",
|
| 59 |
+
"5000",
|
| 60 |
+
"--trainer.logging_frequency",
|
| 61 |
+
"50",
|
| 62 |
+
"--trainer.eval_interval",
|
| 63 |
+
"100",
|
| 64 |
+
"--trainer.gradient_accumulation_steps",
|
| 65 |
+
"1",
|
| 66 |
+
"--trainer.is_resume",
|
| 67 |
+
"true",
|
| 68 |
+
"--run_root_dir",
|
| 69 |
+
"./results/Checkpoints",
|
| 70 |
+
"--run_id",
|
| 71 |
+
"fastumi_pickandplace_qwenPI_329v4",
|
| 72 |
+
"--wandb_project",
|
| 73 |
+
"starVLA_FastUMI_dynamic_1",
|
| 74 |
+
"--wandb_entity",
|
| 75 |
+
"2200011093-peking-university"
|
| 76 |
+
],
|
| 77 |
+
"program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py",
|
| 78 |
+
"codePath": "starVLA/training/train_starvla.py",
|
| 79 |
+
"codePathLocal": "starVLA/training/train_starvla.py",
|
| 80 |
+
"git": {
|
| 81 |
+
"remote": "https://github.com/Kaiwen-Hong/starVLA.git",
|
| 82 |
+
"commit": "15c7d05ddb9af57d0a343b79a1bce9cf8b59b9a5"
|
| 83 |
+
},
|
| 84 |
+
"email": "wangpc@berkeley.edu",
|
| 85 |
+
"root": "./results/Checkpoints/fastumi_pickandplace_qwenPI_329v4/wandb",
|
| 86 |
+
"host": "tams02",
|
| 87 |
+
"executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10",
|
| 88 |
+
"cpu_count": 192,
|
| 89 |
+
"cpu_count_logical": 384,
|
| 90 |
+
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 91 |
+
"gpu_count": 8,
|
| 92 |
+
"disk": {
|
| 93 |
+
"/": {
|
| 94 |
+
"total": "3776651378688",
|
| 95 |
+
"used": "159350890496"
|
| 96 |
+
}
|
| 97 |
+
},
|
| 98 |
+
"memory": {
|
| 99 |
+
"total": "1081552064512"
|
| 100 |
+
},
|
| 101 |
+
"gpu_nvidia": [
|
| 102 |
+
{
|
| 103 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 104 |
+
"memoryTotal": "102641958912",
|
| 105 |
+
"cudaCores": 24064,
|
| 106 |
+
"architecture": "Blackwell",
|
| 107 |
+
"uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 111 |
+
"memoryTotal": "102641958912",
|
| 112 |
+
"cudaCores": 24064,
|
| 113 |
+
"architecture": "Blackwell",
|
| 114 |
+
"uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5"
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 118 |
+
"memoryTotal": "102641958912",
|
| 119 |
+
"cudaCores": 24064,
|
| 120 |
+
"architecture": "Blackwell",
|
| 121 |
+
"uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756"
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 125 |
+
"memoryTotal": "102641958912",
|
| 126 |
+
"cudaCores": 24064,
|
| 127 |
+
"architecture": "Blackwell",
|
| 128 |
+
"uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005"
|
| 129 |
+
},
|
| 130 |
+
{
|
| 131 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 132 |
+
"memoryTotal": "102641958912",
|
| 133 |
+
"cudaCores": 24064,
|
| 134 |
+
"architecture": "Blackwell",
|
| 135 |
+
"uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd"
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 139 |
+
"memoryTotal": "102641958912",
|
| 140 |
+
"cudaCores": 24064,
|
| 141 |
+
"architecture": "Blackwell",
|
| 142 |
+
"uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26"
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 146 |
+
"memoryTotal": "102641958912",
|
| 147 |
+
"cudaCores": 24064,
|
| 148 |
+
"architecture": "Blackwell",
|
| 149 |
+
"uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
|
| 153 |
+
"memoryTotal": "102641958912",
|
| 154 |
+
"cudaCores": 24064,
|
| 155 |
+
"architecture": "Blackwell",
|
| 156 |
+
"uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47"
|
| 157 |
+
}
|
| 158 |
+
],
|
| 159 |
+
"cudaVersion": "13.1",
|
| 160 |
+
"writerId": "yvqkper4i64llzfeix35jg5uyj0abgzb"
|
| 161 |
+
}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"mse_score":0.0019335275515913962,"data_time":0.002223946969024837,"_runtime":12590.341302622,"_step":15000,"action_dit_loss":0.03711752966046333,"_wandb":{"runtime":12590},"learning_rate":2.800000000000001e-06,"epoch":14.55,"_timestamp":1.7751932045113232e+09,"model_time":2.492517018923536}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/logs/debug-core.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-03T01:45:17.076480788Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdq07krtk/port-518615.txt","pid":518615,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-03T01:45:17.077006092Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":518615}
|
| 3 |
+
{"time":"2026-04-03T01:45:17.076992951Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-518615-520095-3190273444/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-03T01:45:17.261245115Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-03T01:45:17.263668848Z","level":"INFO","msg":"handleInformInit: received","streamId":"n5d66d57","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-03T01:45:17.50265569Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n5d66d57","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-03T01:45:22.947932489Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3f6cbhl2hcbe"}
|
| 8 |
+
{"time":"2026-04-03T05:15:08.203439235Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-03T05:15:08.203574854Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-04-03T05:15:08.203666954Z","level":"INFO","msg":"server is shutting down"}
|
| 11 |
+
{"time":"2026-04-03T05:15:08.203768174Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-03T05:15:08.203868004Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-518615-520095-3190273444/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-03T05:15:08.559706484Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-03T01:45:17.263943855Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"}
|
| 2 |
+
{"time":"2026-04-03T01:45:17.502454295Z","level":"INFO","msg":"stream: created new stream","id":"n5d66d57"}
|
| 3 |
+
{"time":"2026-04-03T01:45:17.502549217Z","level":"INFO","msg":"handler: started","stream_id":"n5d66d57"}
|
| 4 |
+
{"time":"2026-04-03T01:45:17.50265022Z","level":"INFO","msg":"stream: started","id":"n5d66d57"}
|
| 5 |
+
{"time":"2026-04-03T01:45:17.50266136Z","level":"INFO","msg":"writer: started","stream_id":"n5d66d57"}
|
| 6 |
+
{"time":"2026-04-03T01:45:17.50267484Z","level":"INFO","msg":"sender: started","stream_id":"n5d66d57"}
|
| 7 |
+
{"time":"2026-04-03T05:15:08.203590884Z","level":"INFO","msg":"stream: closing","id":"n5d66d57"}
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/logs/debug.log
ADDED
|
File without changes
|
fastumi_pickandplace_qwenPI_329v4/wandb/wandb/run-20260403_014517-n5d66d57/run-n5d66d57.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a80ff4e62eff6f1f5aa30efdf45f1916ec31ebbf51ce6a0602d94e5842427f2
|
| 3 |
+
size 5701632
|