diff --git a/.gitattributes b/.gitattributes index b3ae2c80c112dcc59472fe94db580aec054f364e..12eb7a8232ce9d0b22b1dba925295b64d6fcb950 100644 --- a/.gitattributes +++ b/.gitattributes @@ -37,3 +37,9 @@ fastumi_pickandplace_discrete_diffusion_real_0314/wandb/wandb/run-20260314_07354 fastumi_pickandplace_discrete_diffusion_real_0314/wandb/wandb/run-20260314_075218-so4tb51q/run-so4tb51q.wandb filter=lfs diff=lfs merge=lfs -text fastumi_pickandplace_discrete_diffusion_real_0314/wandb/wandb/run-20260314_081115-do490bua/run-do490bua.wandb filter=lfs diff=lfs merge=lfs -text fastumi_pickandplace_discrete_diffusion_real_0314/wandb/wandb/run-20260314_231623-3mz29zso/run-3mz29zso.wandb filter=lfs diff=lfs merge=lfs -text +fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/run-9547dyq0.wandb filter=lfs diff=lfs merge=lfs -text +fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/run-pxng7bwb.wandb filter=lfs diff=lfs merge=lfs -text +fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/run-va5ln8ez.wandb filter=lfs diff=lfs merge=lfs -text +fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/run-hh8947p1.wandb filter=lfs diff=lfs merge=lfs -text +fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/run-uhbkzqjc.wandb filter=lfs diff=lfs merge=lfs -text +fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/run-nyjyyu3l.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/fastumi_pickandplace_qwenPI/checkpoints/steps_10000_pytorch_model.pt b/fastumi_pickandplace_qwenPI/checkpoints/steps_10000_pytorch_model.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e1ce8624ae040e9e0e13bd03f21a38ea15cfeb9 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/checkpoints/steps_10000_pytorch_model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f9faf75dad85aa79ec82e2312a32c46158d01f80341d28bcbfceb1d024200b +size 12444522384 diff --git a/fastumi_pickandplace_qwenPI/checkpoints/steps_15000_pytorch_model.pt b/fastumi_pickandplace_qwenPI/checkpoints/steps_15000_pytorch_model.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae873a6446487922d8d0ff215f94bc5379af5913 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/checkpoints/steps_15000_pytorch_model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4e6f9febe6c1795a63a469f262362f87ce7ef27697246af968fe9a81cb26672 +size 12444522384 diff --git a/fastumi_pickandplace_qwenPI/checkpoints/steps_5000_pytorch_model.pt b/fastumi_pickandplace_qwenPI/checkpoints/steps_5000_pytorch_model.pt new file mode 100644 index 0000000000000000000000000000000000000000..d09f51075675602fcf141a174fee9be43b9cd7c8 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/checkpoints/steps_5000_pytorch_model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b748948f95cbe1ca3cd8c5cefdeb36455303833569a15d9ac695892eb2e043e +size 12444521025 diff --git a/fastumi_pickandplace_qwenPI/config.yaml b/fastumi_pickandplace_qwenPI/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cb6b7dce7206b001a8d249b69a7eba712815dbd --- /dev/null +++ b/fastumi_pickandplace_qwenPI/config.yaml @@ -0,0 +1,70 @@ +datasets: + vla_data: + CoT_prompt: Your task is {instruction}. To identify the key objects for your task. + Locate their bounding boxes in [x1,y1,x2,y2] format. + data_mix: fastumi_pickandplace_ur5_0314 + data_root_dir: playground/Datasets/FastUMI + dataset_py: lerobot_datasets + per_device_batch_size: 8 + video_backend: torchvision_av +framework: + action_model: + action_dim: 10 + add_pos_embed: true + diffusion_model_cfg: + attention_head_dim: 64 + cross_attention_dim: 2048 + dropout: 0.2 + final_dropout: true + input_embedding_dim: 2048 + interleave_self_attention: true + norm_type: ada_norm + num_attention_heads: 32 + num_layers: 36 + output_dim: 1024 + positional_embeddings: null + future_action_window_size: 15 + max_seq_len: 1024 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + num_timestep_buckets: 1000 + past_action_window_size: 0 + state_dim: 10 + name: QwenPI + qwenvl: + attn_implementation: flash_attention_2 + base_vlm: playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action + num_vl_layers: 36 + vl_hidden_dim: 2048 +output_dir: ./results/Checkpoints/fastumi_pickandplace_qwenPI +run_id: fastumi_pickandplace_qwenPI +run_root_dir: ./results/Checkpoints +seed: 42 +trainer: + eval_interval: 100 + freeze_modules: null + gradient_accumulation_steps: 1 + gradient_clipping: 1.0 + is_resume: true + learning_rate: + action_model: 0.0001 + base: 2.5e-05 + qwen_vl_interface: 1.0e-05 + logging_frequency: 50 + lr_scheduler_type: cosine_with_min_lr + max_train_steps: 20000 + num_warmup_steps: 5000 + optimizer: + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + weight_decay: 1.0e-08 + save_interval: 5000 + scheduler_specific_kwargs: + min_lr: 1.0e-06 +wandb_entity: 2200011093-peking-university +wandb_project: starVLA_FastUMI diff --git a/fastumi_pickandplace_qwenPI/dataset_statistics.json b/fastumi_pickandplace_qwenPI/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..4f42cf878a30ce6cae214812c811a9ad6cce17b9 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/dataset_statistics.json @@ -0,0 +1,178 @@ +{ + "new_embodiment": { + "action": { + "mean": [ + -0.00017731482512317598, + 6.326655420707539e-05, + -0.0003823576553259045, + 0.9999275803565979, + 0.00015219292254187167, + -7.193408237071708e-05, + -0.00015269513824023306, + 0.9998728632926941, + -0.0001340110320597887, + 0.23157350718975067 + ], + "std": [ + 0.0029254474211484194, + 0.00266967061907053, + 0.004045490175485611, + 0.0004587694129540108, + 0.011643894948065281, + 0.0005491027259267867, + 0.011645006015896797, + 0.0008708133827583353, + 0.011052136309444904, + 0.24192321300506592 + ], + "max": [ + 0.011658241041004658, + 0.01042273361235857, + 0.0110545065253973, + 1.0000001192092896, + 0.08344583958387375, + 0.0038863078225404024, + 0.08254292607307434, + 1.0000001192092896, + 0.079257071018219, + 0.5245640277862549 + ], + "min": [ + -0.013589359819889069, + -0.013344451785087585, + -0.012630008161067963, + 0.9965112805366516, + -0.08266565203666687, + -0.01173458807170391, + -0.08334038406610489, + 0.9939950704574585, + -0.07899841666221619, + 0.009166000410914421 + ], + "q01": [ + -0.007945208344608545, + -0.007611272423528135, + -0.009110004445537924, + 0.9968565315008163, + -0.000308865460101515, + -0.0032842739787884052, + -0.07796837285161018, + 0.9940040111541748, + -0.07265709199011326, + 0.009802999906241894 + ], + "q99": [ + 0.007261873693205412, + 0.007667668941430744, + 0.008452737815678119, + 1.0, + 0.07798104047775269, + 0.0002730701782274991, + 0.0003088487408240318, + 1.0, + 0.0004576626507332527, + 0.5240129828453064 + ], + "mask": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + false + ], + "norm_modes": [ + "min_max", + "min_max", + "min_max", + "none", + "none", + "none", + "none", + "none", + "none", + "binary" + ] + }, + "state": { + "mean": [ + 0.34298810362815857, + -0.29047951102256775, + 0.08274945616722107, + 0.9994291067123413, + -0.0020131091587245464, + -0.0012421202845871449, + 0.002004049951210618, + 0.9983507394790649, + 0.0019875187426805496, + 0.22578883171081543 + ], + "std": [ + 0.04991065338253975, + 0.0422644317150116, + 0.06237703189253807, + 0.00019692142086756705, + 0.039019376039505005, + 0.0012851571664214134, + 0.03903917223215103, + 0.00019175464691510824, + 0.03722655400633812, + 0.24103344976902008 + ], + "max": [ + 0.49783000349998474, + -0.10034999996423721, + 0.32138198614120483, + 0.9993224143981934, + 0.04183037206530571, + 0.004732622765004635, + 0.04235748574137688, + 0.9986677765846252, + 0.03970242291688919, + 0.5245640277862549 + ], + "min": [ + 0.19434399902820587, + -0.44425100088119507, + 0.0019030000548809767, + 0.999085009098053, + -0.042528580874204636, + -0.005806926172226667, + -0.04183799773454666, + 0.9984964728355408, + -0.03961476683616638, + 0.009166000410914421 + ], + "q01": [ + 0.23710880234837534, + -0.3849276554584503, + 0.00841386997140944, + 0.9991415911912918, + -0.041123956106603146, + -0.004370019193738699, + -0.04099516797810793, + 0.9984980225563049, + -0.03880806788802147, + 0.009802999906241894 + ], + "q99": [ + 0.46919367551803587, + -0.1585090310871601, + 0.2566446015238762, + 0.999306601881981, + 0.04096023611724377, + 0.002124080888461314, + 0.04100460018962622, + 0.9986270666122437, + 0.03915046207606793, + 0.5240129828453064 + ] + }, + "num_transitions": 26330, + "num_trajectories": 300 + } +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/summary.jsonl b/fastumi_pickandplace_qwenPI/summary.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..502ebe934436b2ca8f4dbd60020c647fece4de8e --- /dev/null +++ b/fastumi_pickandplace_qwenPI/summary.jsonl @@ -0,0 +1,3 @@ +{"steps": 5000} +{"steps": 10000} +{"steps": 15000} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3bfd0470fba31e8b0866b4b40924fae10df6434b --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2026-03-15T07:22:13.699370785Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-15T07:22:14.143677678Z","level":"INFO","msg":"stream: created new stream","id":"nyjyyu3l"} +{"time":"2026-03-15T07:22:14.143798108Z","level":"INFO","msg":"handler: started","stream_id":"nyjyyu3l"} +{"time":"2026-03-15T07:22:14.143934497Z","level":"INFO","msg":"stream: started","id":"nyjyyu3l"} +{"time":"2026-03-15T07:22:14.143996517Z","level":"INFO","msg":"writer: started","stream_id":"nyjyyu3l"} +{"time":"2026-03-15T07:22:14.143999787Z","level":"INFO","msg":"sender: started","stream_id":"nyjyyu3l"} +{"time":"2026-03-15T18:13:36.019181495Z","level":"INFO","msg":"stream: closing","id":"nyjyyu3l"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/config.yaml b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45a7f28a78ff62d24d9fff1cc2dc7ae9927ebb01 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/config.yaml @@ -0,0 +1,168 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + 9rs4jt49lzudq17zfe4jqbilhwm7rk8l: + args: + - --config_yaml + - ./examples/calvin/train_files/starvla_train_calvin.yaml + - --framework.name + - QwenPI + - --framework.qwenvl.base_vlm + - playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action + - --framework.qwenvl.attn_implementation + - flash_attention_2 + - --framework.action_model.action_dim + - "10" + - --framework.action_model.state_dim + - "10" + - --framework.action_model.future_action_window_size + - "15" + - --framework.action_model.past_action_window_size + - "0" + - --framework.action_model.action_hidden_dim + - "1024" + - --framework.action_model.hidden_size + - "1024" + - --framework.action_model.action_model_type + - DiT-B + - --framework.action_model.add_pos_embed + - "True" + - --framework.action_model.max_seq_len + - "1024" + - --framework.action_model.noise_beta_alpha + - "1.5" + - --framework.action_model.noise_beta_beta + - "1.0" + - --framework.action_model.noise_s + - "0.999" + - --framework.action_model.num_timestep_buckets + - "1000" + - --framework.action_model.num_inference_timesteps + - "4" + - --framework.action_model.num_target_vision_tokens + - "32" + - --datasets.vla_data.data_root_dir + - playground/Datasets/FastUMI + - --datasets.vla_data.data_mix + - fastumi_pickandplace_real_0307 + - --datasets.vla_data.per_device_batch_size + - "8" + - --datasets.vla_data.video_backend + - torchvision_av + - --trainer.freeze_modules + - "" + - --trainer.max_train_steps + - "20000" + - --trainer.save_interval + - "5000" + - --trainer.logging_frequency + - "50" + - --trainer.eval_interval + - "100" + - --trainer.gradient_accumulation_steps + - "1" + - --trainer.is_resume + - "true" + - --run_root_dir + - ./results/Checkpoints + - --run_id + - fastumi_pickandplace_qwenPI + - --wandb_project + - starVLA_FastUMI + - --wandb_entity + - 2200011093-peking-university + codePath: starVLA/training/train_starvla.py + codePathLocal: starVLA/training/train_starvla.py + cpu_count: 192 + cpu_count_logical: 384 + cudaVersion: "13.1" + disk: + /: + total: "3776651378688" + used: "136350265344" + email: wangpc@berkeley.edu + executable: /home/wangpc/miniconda3/envs/starVLA/bin/python3.10 + git: + commit: 66b43863ede17a0e3081f822346e95cec52816d0 + remote: https://github.com/Kaiwen-Hong/starVLA.git + gpu: NVIDIA RTX PRO 6000 Blackwell Server Edition + gpu_count: 8 + gpu_nvidia: + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-1f3c0889-b740-5143-e064-afeb49245756 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-49955a45-509a-e8af-a468-b9ef0449b005 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-d16b5acf-a100-2d1b-8138-1661892f3d26 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-747dc32b-b025-76e9-697d-fd33ef441b47 + host: tams02 + memory: + total: "1081550508032" + os: Linux-6.8.0-94-generic-x86_64-with-glibc2.39 + program: /scratch/wangpc/starVLA/starVLA/training/train_starvla.py + python: CPython 3.10.19 + root: ./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb + startedAt: "2026-03-13T04:12:22.513938Z" + writerId: 9rs4jt49lzudq17zfe4jqbilhwm7rk8l + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + - 80 + - 83 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + - 80 + - 83 + "3": + - 13 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.57.0 + "12": 0.25.0 + "13": linux-x86_64 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/output.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a4d5617650dc54eebcdc0ed6f5e63ee8ababced1 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/output.log @@ -0,0 +1,25 @@ +03/13 [04:12:23] INFO  | >> ***** Training Configuration ***** ]8;id=98246;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=229258;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#338\338]8;;\ +  INFO  | >> Total optimization steps = 20000 ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#339\339]8;;\ +  INFO  | >> Per device batch size = 8 ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#340\340]8;;\ +  INFO  | >> Gradient accumulation steps = 1 ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\341]8;;\ +  INFO  | >> Total batch size = 64 ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\342]8;;\ + 0%| | 1/20000 [00:04<22:16:25, 4.01s/it, data_times=0.608, model_times=3.402]Traceback (most recent call last): + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 440, in + main(cfg) + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 410, in main + trainer.train() + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train + step_metrics = self._train_step(batch_vla) + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 363, in _train_step + "action_dit_loss": action_loss.item(), +KeyboardInterrupt +[rank0]: Traceback (most recent call last): +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 440, in +[rank0]: main(cfg) +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 410, in main +[rank0]: trainer.train() +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train +[rank0]: step_metrics = self._train_step(batch_vla) +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 363, in _train_step +[rank0]: "action_dit_loss": action_loss.item(), +[rank0]: KeyboardInterrupt diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/requirements.txt b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..33491c4b7fdcd2a21a3c18e0df523fddefb31ef2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/requirements.txt @@ -0,0 +1,154 @@ +starVLA==1.0.1 +kiwisolver==1.4.9 +scipy==1.15.3 +pyarrow==14.0.1 +protobuf==6.33.5 +platformdirs==4.9.4 +mdurl==0.1.2 +Jinja2==3.1.6 +torchvision==0.22.0+cu128 +exceptiongroup==1.3.1 +nvidia-cusparselt-cu12==0.6.3 +markdown-it-py==4.0.0 +timm==1.0.25 +nvidia-nvjitlink-cu12==12.8.61 +urllib3==2.6.3 +numpydantic==1.6.9 +pillow==12.1.1 +json-numpy==2.1.1 +fastparquet==2024.11.0 +contourpy==1.3.2 +tensorboard-data-server==0.7.2 +albumentations==1.4.18 +deepspeed==0.16.9 +ImageIO==2.37.2 +huggingface_hub==0.36.2 +hjson==3.1.0 +tqdm==4.67.3 +idna==3.11 +packaging==25.0 +python-dateutil==2.9.0.post0 +annotated-types==0.7.0 +regex==2026.2.28 +snntorch==0.9.4 +cramjam==2.11.0 +importlib_metadata==8.7.1 +torch==2.7.0+cu128 +yacs==0.1.8 +msgpack==1.1.2 +h11==0.16.0 +nvidia-cuda-runtime-cu12==12.8.57 +typing_extensions==4.15.0 +scikit-image==0.25.2 +mpmath==1.3.0 +einops==0.8.2 +wandb==0.25.0 +anyio==4.12.1 +flash_attn==2.7.4.post1 +websockets==15.0.1 +requests==2.32.5 +accelerate==1.5.2 +nvidia-cuda-cupti-cu12==12.8.57 +nvidia-cuda-nvrtc-cu12==12.8.61 +PyYAML==6.0.3 +absl-py==2.4.0 +httpx==0.28.1 +pipablepytorch3d==0.7.6 +nvidia-cublas-cu12==12.8.3.14 +decord==0.6.0 +ninja==1.13.0 +albucore==0.0.17 +pyparsing==3.3.2 +triton==3.3.0 +Markdown==3.10.2 +Pygments==2.19.2 +pydantic==2.10.6 +tabulate==0.10.0 +termcolor==3.3.0 +zipp==3.23.0 +Werkzeug==3.1.6 +sympy==1.14.0 +debugpy==1.8.20 +certifi==2026.2.25 +websocket==0.2.1 +fonttools==4.61.1 +transformers==4.57.0 +av==12.3.0 +transformers-stream-generator==0.0.4 +nvidia-cudnn-cu12==9.7.1.26 +nvidia-curand-cu12==10.3.9.55 +six==1.17.0 +fvcore==0.1.5.post20221221 +matplotlib==3.10.8 +lazy_loader==0.4 +nvidia-nccl-cu12==2.26.2 +diffusers==0.37.0 +tifffile==2025.5.10 +GitPython==3.1.46 +tokenizers==0.22.2 +eval_type_backport==0.3.1 +nvidia-cufile-cu12==1.13.0.11 +numpy==1.26.4 +filelock==3.25.0 +fsspec==2026.2.0 +nvidia-cusolver-cu12==11.7.2.55 +MarkupSafe==3.0.3 +tyro==1.0.8 +pydantic_core==2.27.2 +portalocker==3.2.0 +qwen-vl-utils==0.0.14 +click==8.3.1 +tiktoken==0.12.0 +smmap==5.0.2 +nvidia-nvtx-cu12==12.8.55 +pytz==2026.1.post1 +rich==14.2.0 +charset-normalizer==3.4.4 +tensorboard==2.20.0 +zope.event==6.1 +zope.interface==8.2 +networkx==3.4.2 +mpi4py==4.1.1 +gitdb==4.0.12 +safetensors==0.7.0 +typeguard==4.5.1 +py-cpuinfo==9.0.0 +websocket-client==1.8.0 +hf-xet==1.3.2 +wheel==0.46.3 +antlr4-python3-runtime==4.9.3 +gevent==25.9.1 +setuptools==80.9.0 +torchaudio==2.7.0+cu128 +nvidia-cufft-cu12==11.3.3.41 +greenlet==3.3.2 +docstring_parser==0.17.0 +iopath==0.1.10 +cycler==0.12.1 +tzdata==2025.3 +psutil==7.2.2 +grpcio==1.78.0 +nvidia-cusparse-cu12==12.5.7.53 +sentry-sdk==2.54.0 +httpcore==1.0.9 +opencv-python-headless==4.11.0.86 +omegaconf==2.3.0 +pandas==2.3.3 +eva-decord==0.6.1 +pip==26.0.1 +inflect==7.3.1 +jaraco.text==3.12.1 +autocommand==2.2.2 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +zipp==3.19.2 +more-itertools==10.3.0 +tomli==2.0.1 +typing_extensions==4.12.2 +typeguard==4.3.0 +wheel==0.45.1 +platformdirs==4.2.2 +importlib_metadata==8.0.0 +jaraco.collections==5.1.0 +packaging==24.2 +jaraco.context==5.3.0 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/wandb-metadata.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..39475f2ec88c5b72c27c28111cfd6e1041fdd8c5 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/wandb-metadata.json @@ -0,0 +1,159 @@ +{ + "os": "Linux-6.8.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-03-13T04:12:22.513938Z", + "args": [ + "--config_yaml", + "./examples/calvin/train_files/starvla_train_calvin.yaml", + "--framework.name", + "QwenPI", + "--framework.qwenvl.base_vlm", + "playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action", + "--framework.qwenvl.attn_implementation", + "flash_attention_2", + "--framework.action_model.action_dim", + "10", + "--framework.action_model.state_dim", + "10", + "--framework.action_model.future_action_window_size", + "15", + "--framework.action_model.past_action_window_size", + "0", + "--framework.action_model.action_hidden_dim", + "1024", + "--framework.action_model.hidden_size", + "1024", + "--framework.action_model.action_model_type", + "DiT-B", + "--framework.action_model.add_pos_embed", + "True", + "--framework.action_model.max_seq_len", + "1024", + "--framework.action_model.noise_beta_alpha", + "1.5", + "--framework.action_model.noise_beta_beta", + "1.0", + "--framework.action_model.noise_s", + "0.999", + "--framework.action_model.num_timestep_buckets", + "1000", + "--framework.action_model.num_inference_timesteps", + "4", + "--framework.action_model.num_target_vision_tokens", + "32", + "--datasets.vla_data.data_root_dir", + "playground/Datasets/FastUMI", + "--datasets.vla_data.data_mix", + "fastumi_pickandplace_real_0307", + "--datasets.vla_data.per_device_batch_size", + "8", + "--datasets.vla_data.video_backend", + "torchvision_av", + "--trainer.freeze_modules", + "", + "--trainer.max_train_steps", + "20000", + "--trainer.save_interval", + "5000", + "--trainer.logging_frequency", + "50", + "--trainer.eval_interval", + "100", + "--trainer.gradient_accumulation_steps", + "1", + "--trainer.is_resume", + "true", + "--run_root_dir", + "./results/Checkpoints", + "--run_id", + "fastumi_pickandplace_qwenPI", + "--wandb_project", + "starVLA_FastUMI", + "--wandb_entity", + "2200011093-peking-university" + ], + "program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", + "codePath": "starVLA/training/train_starvla.py", + "codePathLocal": "starVLA/training/train_starvla.py", + "git": { + "remote": "https://github.com/Kaiwen-Hong/starVLA.git", + "commit": "66b43863ede17a0e3081f822346e95cec52816d0" + }, + "email": "wangpc@berkeley.edu", + "root": "./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb", + "host": "tams02", + "executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10", + "cpu_count": 192, + "cpu_count_logical": 384, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "gpu_count": 8, + "disk": { + "/": { + "total": "3776651378688", + "used": "136350265344" + } + }, + "memory": { + "total": "1081550508032" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47" + } + ], + "cudaVersion": "13.1", + "writerId": "9rs4jt49lzudq17zfe4jqbilhwm7rk8l" +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/wandb-summary.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f60bccc799a769e529cf191d8e66da19a5816ef1 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":6},"_runtime":6} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/logs/debug-core.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..2203404b0b8aab6314abbde90488a6d5d47e721a --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2026-03-13T04:12:22.578275025Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp6hhsvuc9/port-1882769.txt","pid":1882769,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-13T04:12:22.579072958Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1882769} +{"time":"2026-03-13T04:12:22.579064158Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1882769-1889351-4240563926/socket","Net":"unix"}} +{"time":"2026-03-13T04:12:22.754310046Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-13T04:12:22.757113104Z","level":"INFO","msg":"handleInformInit: received","streamId":"hshxz32s","id":"1(@)"} +{"time":"2026-03-13T04:12:23.096003393Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"hshxz32s","id":"1(@)"} +{"time":"2026-03-13T04:12:28.429538362Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"af6ag242d2oa"} +{"time":"2026-03-13T04:12:29.993215437Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-13T04:12:29.993286866Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-13T04:12:29.993282846Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-13T04:12:29.993379816Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-13T04:12:29.993391266Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1882769-1889351-4240563926/socket","Net":"unix"}} +{"time":"2026-03-13T04:12:30.299408944Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/logs/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..93a0eac6491c2b23e04b07bf483cfe8dd6194c72 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2026-03-13T04:12:22.757194123Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-13T04:12:23.095857965Z","level":"INFO","msg":"stream: created new stream","id":"hshxz32s"} +{"time":"2026-03-13T04:12:23.095946134Z","level":"INFO","msg":"handler: started","stream_id":"hshxz32s"} +{"time":"2026-03-13T04:12:23.095998773Z","level":"INFO","msg":"stream: started","id":"hshxz32s"} +{"time":"2026-03-13T04:12:23.096020003Z","level":"INFO","msg":"writer: started","stream_id":"hshxz32s"} +{"time":"2026-03-13T04:12:23.096030263Z","level":"INFO","msg":"sender: started","stream_id":"hshxz32s"} +{"time":"2026-03-13T04:12:29.993293716Z","level":"INFO","msg":"stream: closing","id":"hshxz32s"} +{"time":"2026-03-13T04:12:30.26757045Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/logs/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/run-hshxz32s.wandb b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/run-hshxz32s.wandb new file mode 100644 index 0000000000000000000000000000000000000000..30977402b08eb894463b83d7de7a007255879a32 Binary files /dev/null and b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_041222-hshxz32s/run-hshxz32s.wandb differ diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/config.yaml b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea2069650993d08b2a159edb0c1a5591aaaf193d --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/config.yaml @@ -0,0 +1,169 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + s3rvfu8shkj8mzdb09dke7i4kcnoimei: + args: + - --config_yaml + - ./examples/calvin/train_files/starvla_train_calvin.yaml + - --framework.name + - QwenPI + - --framework.qwenvl.base_vlm + - playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action + - --framework.qwenvl.attn_implementation + - flash_attention_2 + - --framework.action_model.action_dim + - "10" + - --framework.action_model.state_dim + - "10" + - --framework.action_model.future_action_window_size + - "15" + - --framework.action_model.past_action_window_size + - "0" + - --framework.action_model.action_hidden_dim + - "1024" + - --framework.action_model.hidden_size + - "1024" + - --framework.action_model.action_model_type + - DiT-B + - --framework.action_model.add_pos_embed + - "True" + - --framework.action_model.max_seq_len + - "1024" + - --framework.action_model.noise_beta_alpha + - "1.5" + - --framework.action_model.noise_beta_beta + - "1.0" + - --framework.action_model.noise_s + - "0.999" + - --framework.action_model.num_timestep_buckets + - "1000" + - --framework.action_model.num_inference_timesteps + - "4" + - --framework.action_model.num_target_vision_tokens + - "32" + - --datasets.vla_data.data_root_dir + - playground/Datasets/FastUMI + - --datasets.vla_data.data_mix + - fastumi_pickandplace_real_0307 + - --datasets.vla_data.per_device_batch_size + - "8" + - --datasets.vla_data.video_backend + - torchvision_av + - --trainer.freeze_modules + - "" + - --trainer.max_train_steps + - "20000" + - --trainer.save_interval + - "5000" + - --trainer.logging_frequency + - "50" + - --trainer.eval_interval + - "100" + - --trainer.gradient_accumulation_steps + - "1" + - --trainer.is_resume + - "true" + - --run_root_dir + - ./results/Checkpoints + - --run_id + - fastumi_pickandplace_qwenPI + - --wandb_project + - starVLA_FastUMI + - --wandb_entity + - 2200011093-peking-university + codePath: starVLA/training/train_starvla.py + codePathLocal: starVLA/training/train_starvla.py + cpu_count: 192 + cpu_count_logical: 384 + cudaVersion: "13.1" + disk: + /: + total: "3776651378688" + used: "136342257664" + email: wangpc@berkeley.edu + executable: /home/wangpc/miniconda3/envs/starVLA/bin/python3.10 + git: + commit: 66b43863ede17a0e3081f822346e95cec52816d0 + remote: https://github.com/Kaiwen-Hong/starVLA.git + gpu: NVIDIA RTX PRO 6000 Blackwell Server Edition + gpu_count: 8 + gpu_nvidia: + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-1f3c0889-b740-5143-e064-afeb49245756 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-49955a45-509a-e8af-a468-b9ef0449b005 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-d16b5acf-a100-2d1b-8138-1661892f3d26 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-747dc32b-b025-76e9-697d-fd33ef441b47 + host: tams02 + memory: + total: "1081550508032" + os: Linux-6.8.0-94-generic-x86_64-with-glibc2.39 + program: /scratch/wangpc/starVLA/starVLA/training/train_starvla.py + python: CPython 3.10.19 + root: ./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb + startedAt: "2026-03-13T05:21:28.850487Z" + writerId: s3rvfu8shkj8mzdb09dke7i4kcnoimei + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + - 80 + - 83 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + - 80 + - 83 + "3": + - 13 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.57.0 + "12": 0.25.0 + "13": linux-x86_64 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/output.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b494e5a5e097a53b2411c6aa711b7bbd9ba79261 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/output.log @@ -0,0 +1,90 @@ +03/13 [05:21:29] INFO  | >> ***** Training Configuration ***** ]8;id=98246;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=229258;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#338\338]8;;\ +  INFO  | >> Total optimization steps = 20000 ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#339\339]8;;\ +  INFO  | >> Per device batch size = 8 ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#340\340]8;;\ +  INFO  | >> Gradient accumulation steps = 1 ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\341]8;;\ +  INFO  | >> Total batch size = 64 ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\342]8;;\ + 2%|█ | 500/20000 [21:07<13:36:04, 2.51s/it, data_times=0.005, model_times=2.491] +03/13 [05:23:38] INFO  | >> Step 50, Loss: {'action_dit_loss': 202009.875, 'data_time': ]8;id=376417;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=888662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016733817756175995, 'model_time': 2.527299334295094, 'learning_rate':   +  1.0000000000000001e-07, 'epoch': 0.14})   +03/13 [05:25:44] INFO  | >> Step 100, Loss: {'action_dit_loss': 133267.859375, 'mse_score': ]8;id=45561;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=765179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.36893367767334, 'data_time': 0.0032478091306984425, 'model_time':   +  2.539055143017322, 'learning_rate': 2.0000000000000002e-07, 'epoch':   +  0.29})   +03/13 [05:27:51] INFO  | >> Step 150, Loss: {'action_dit_loss': 2462.653076171875, 'data_time': ]8;id=396922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=82627;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015402119606733322, 'model_time': 2.4931312440894544, 'learning_rate':   +  3.0000000000000004e-07, 'epoch': 0.43})   +03/13 [05:29:57] INFO  | >> Step 200, Loss: {'action_dit_loss': 2822.628662109375, 'mse_score': ]8;id=648564;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=928463;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  15.090812683105469, 'data_time': 0.0030169920064508915, 'model_time':   +  2.534589695278555, 'learning_rate': 4.0000000000000003e-07, 'epoch':   +  0.57})   +03/13 [05:32:03] INFO  | >> Step 250, Loss: {'action_dit_loss': 375.4444580078125, 'data_time': ]8;id=738797;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=72933;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0033243270590901375, 'model_time': 2.506985930260271, 'learning_rate':   +  5.000000000000001e-07, 'epoch': 0.72})   +03/13 [05:34:10] INFO  | >> Step 300, Loss: {'action_dit_loss': 216.59744262695312, 'mse_score': ]8;id=303445;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=83667;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  6.797264862060547, 'data_time': 0.002378092147409916, 'model_time':   +  2.4954333789646626, 'learning_rate': 6.000000000000001e-07, 'epoch':   +  0.86})   +03/13 [05:36:17] INFO  | >> Step 350, Loss: {'action_dit_loss': 118.1450424194336, 'data_time': ]8;id=398591;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=291476;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004711804911494255, 'model_time': 2.535939588211477, 'learning_rate':   +  7.000000000000001e-07, 'epoch': 1.01})   +03/13 [05:38:24] INFO  | >> Step 400, Loss: {'action_dit_loss': 32.82866287231445, 'mse_score': ]8;id=170555;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=388162;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  3.5609485626220705, 'data_time': 0.0002001640386879444, 'model_time':   +  2.75067617604509, 'learning_rate': 8.000000000000001e-07, 'epoch': 1.15})   +03/13 [05:40:30] INFO  | >> Step 450, Loss: {'action_dit_loss': 11.134794235229492, 'data_time': ]8;id=735911;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=982153;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00020384322851896286, 'model_time': 2.523681683000177, 'learning_rate':   +  9.000000000000001e-07, 'epoch': 1.29})   +03/13 [05:42:37] INFO  | >> Step 500, Loss: {'action_dit_loss': 46.9814453125, 'mse_score': ]8;id=665822;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=179451;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  7.125044250488282, 'data_time': 0.004552784841507673, 'model_time':   +  2.4910148638300598, 'learning_rate': 1.0000000000000002e-06, 'epoch':   +  1.44})   +03/13 [05:44:43] INFO  | >> Step 550, Loss: {'action_dit_loss': 170.627685546875, 'data_time': ]8;id=484714;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=397887;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.000180800911039114, 'model_time': 2.5395604190416634, 'learning_rate':   +  1.1e-06, 'epoch': 1.58})   +03/13 [05:46:50] INFO  | >> Step 600, Loss: {'action_dit_loss': 914.7533569335938, 'mse_score': ]8;id=584004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=230283;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  12.847688293457031, 'data_time': 0.002079134341329336, 'model_time':   +  2.504630303941667, 'learning_rate': 1.2000000000000002e-06, 'epoch':   +  1.72})   + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 440, in + main(cfg) + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 410, in main + trainer.train() + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train + step_metrics = self._train_step(batch_vla) + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 354, in _train_step + self.accelerator.backward(total_loss) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward + self.deepspeed_engine_wrapped.backward(loss, **kwargs) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 275, in backward + self.engine.step() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2382, in step + self.tput_timer.stop(global_step=self.is_gradient_accumulation_boundary(), report_speed=report_progress) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/timer.py", line 256, in stop + get_accelerator().synchronize() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/accelerator/cuda_accelerator.py", line 79, in synchronize + return torch.cuda.synchronize(device_index) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1040, in synchronize + return torch._C._cuda_synchronize() +KeyboardInterrupt +[rank0]: Traceback (most recent call last): +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 440, in +[rank0]: main(cfg) +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 410, in main +[rank0]: trainer.train() +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train +[rank0]: step_metrics = self._train_step(batch_vla) +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 354, in _train_step +[rank0]: self.accelerator.backward(total_loss) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward +[rank0]: self.deepspeed_engine_wrapped.backward(loss, **kwargs) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 275, in backward +[rank0]: self.engine.step() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2382, in step +[rank0]: self.tput_timer.stop(global_step=self.is_gradient_accumulation_boundary(), report_speed=report_progress) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/timer.py", line 256, in stop +[rank0]: get_accelerator().synchronize() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/accelerator/cuda_accelerator.py", line 79, in synchronize +[rank0]: return torch.cuda.synchronize(device_index) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1040, in synchronize +[rank0]: return torch._C._cuda_synchronize() +[rank0]: KeyboardInterrupt diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/requirements.txt b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..33491c4b7fdcd2a21a3c18e0df523fddefb31ef2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/requirements.txt @@ -0,0 +1,154 @@ +starVLA==1.0.1 +kiwisolver==1.4.9 +scipy==1.15.3 +pyarrow==14.0.1 +protobuf==6.33.5 +platformdirs==4.9.4 +mdurl==0.1.2 +Jinja2==3.1.6 +torchvision==0.22.0+cu128 +exceptiongroup==1.3.1 +nvidia-cusparselt-cu12==0.6.3 +markdown-it-py==4.0.0 +timm==1.0.25 +nvidia-nvjitlink-cu12==12.8.61 +urllib3==2.6.3 +numpydantic==1.6.9 +pillow==12.1.1 +json-numpy==2.1.1 +fastparquet==2024.11.0 +contourpy==1.3.2 +tensorboard-data-server==0.7.2 +albumentations==1.4.18 +deepspeed==0.16.9 +ImageIO==2.37.2 +huggingface_hub==0.36.2 +hjson==3.1.0 +tqdm==4.67.3 +idna==3.11 +packaging==25.0 +python-dateutil==2.9.0.post0 +annotated-types==0.7.0 +regex==2026.2.28 +snntorch==0.9.4 +cramjam==2.11.0 +importlib_metadata==8.7.1 +torch==2.7.0+cu128 +yacs==0.1.8 +msgpack==1.1.2 +h11==0.16.0 +nvidia-cuda-runtime-cu12==12.8.57 +typing_extensions==4.15.0 +scikit-image==0.25.2 +mpmath==1.3.0 +einops==0.8.2 +wandb==0.25.0 +anyio==4.12.1 +flash_attn==2.7.4.post1 +websockets==15.0.1 +requests==2.32.5 +accelerate==1.5.2 +nvidia-cuda-cupti-cu12==12.8.57 +nvidia-cuda-nvrtc-cu12==12.8.61 +PyYAML==6.0.3 +absl-py==2.4.0 +httpx==0.28.1 +pipablepytorch3d==0.7.6 +nvidia-cublas-cu12==12.8.3.14 +decord==0.6.0 +ninja==1.13.0 +albucore==0.0.17 +pyparsing==3.3.2 +triton==3.3.0 +Markdown==3.10.2 +Pygments==2.19.2 +pydantic==2.10.6 +tabulate==0.10.0 +termcolor==3.3.0 +zipp==3.23.0 +Werkzeug==3.1.6 +sympy==1.14.0 +debugpy==1.8.20 +certifi==2026.2.25 +websocket==0.2.1 +fonttools==4.61.1 +transformers==4.57.0 +av==12.3.0 +transformers-stream-generator==0.0.4 +nvidia-cudnn-cu12==9.7.1.26 +nvidia-curand-cu12==10.3.9.55 +six==1.17.0 +fvcore==0.1.5.post20221221 +matplotlib==3.10.8 +lazy_loader==0.4 +nvidia-nccl-cu12==2.26.2 +diffusers==0.37.0 +tifffile==2025.5.10 +GitPython==3.1.46 +tokenizers==0.22.2 +eval_type_backport==0.3.1 +nvidia-cufile-cu12==1.13.0.11 +numpy==1.26.4 +filelock==3.25.0 +fsspec==2026.2.0 +nvidia-cusolver-cu12==11.7.2.55 +MarkupSafe==3.0.3 +tyro==1.0.8 +pydantic_core==2.27.2 +portalocker==3.2.0 +qwen-vl-utils==0.0.14 +click==8.3.1 +tiktoken==0.12.0 +smmap==5.0.2 +nvidia-nvtx-cu12==12.8.55 +pytz==2026.1.post1 +rich==14.2.0 +charset-normalizer==3.4.4 +tensorboard==2.20.0 +zope.event==6.1 +zope.interface==8.2 +networkx==3.4.2 +mpi4py==4.1.1 +gitdb==4.0.12 +safetensors==0.7.0 +typeguard==4.5.1 +py-cpuinfo==9.0.0 +websocket-client==1.8.0 +hf-xet==1.3.2 +wheel==0.46.3 +antlr4-python3-runtime==4.9.3 +gevent==25.9.1 +setuptools==80.9.0 +torchaudio==2.7.0+cu128 +nvidia-cufft-cu12==11.3.3.41 +greenlet==3.3.2 +docstring_parser==0.17.0 +iopath==0.1.10 +cycler==0.12.1 +tzdata==2025.3 +psutil==7.2.2 +grpcio==1.78.0 +nvidia-cusparse-cu12==12.5.7.53 +sentry-sdk==2.54.0 +httpcore==1.0.9 +opencv-python-headless==4.11.0.86 +omegaconf==2.3.0 +pandas==2.3.3 +eva-decord==0.6.1 +pip==26.0.1 +inflect==7.3.1 +jaraco.text==3.12.1 +autocommand==2.2.2 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +zipp==3.19.2 +more-itertools==10.3.0 +tomli==2.0.1 +typing_extensions==4.12.2 +typeguard==4.3.0 +wheel==0.45.1 +platformdirs==4.2.2 +importlib_metadata==8.0.0 +jaraco.collections==5.1.0 +packaging==24.2 +jaraco.context==5.3.0 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/wandb-metadata.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ea2ac1277077873ee7d2743984ae3aedecb6bf44 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/wandb-metadata.json @@ -0,0 +1,159 @@ +{ + "os": "Linux-6.8.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-03-13T05:21:28.850487Z", + "args": [ + "--config_yaml", + "./examples/calvin/train_files/starvla_train_calvin.yaml", + "--framework.name", + "QwenPI", + "--framework.qwenvl.base_vlm", + "playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action", + "--framework.qwenvl.attn_implementation", + "flash_attention_2", + "--framework.action_model.action_dim", + "10", + "--framework.action_model.state_dim", + "10", + "--framework.action_model.future_action_window_size", + "15", + "--framework.action_model.past_action_window_size", + "0", + "--framework.action_model.action_hidden_dim", + "1024", + "--framework.action_model.hidden_size", + "1024", + "--framework.action_model.action_model_type", + "DiT-B", + "--framework.action_model.add_pos_embed", + "True", + "--framework.action_model.max_seq_len", + "1024", + "--framework.action_model.noise_beta_alpha", + "1.5", + "--framework.action_model.noise_beta_beta", + "1.0", + "--framework.action_model.noise_s", + "0.999", + "--framework.action_model.num_timestep_buckets", + "1000", + "--framework.action_model.num_inference_timesteps", + "4", + "--framework.action_model.num_target_vision_tokens", + "32", + "--datasets.vla_data.data_root_dir", + "playground/Datasets/FastUMI", + "--datasets.vla_data.data_mix", + "fastumi_pickandplace_real_0307", + "--datasets.vla_data.per_device_batch_size", + "8", + "--datasets.vla_data.video_backend", + "torchvision_av", + "--trainer.freeze_modules", + "", + "--trainer.max_train_steps", + "20000", + "--trainer.save_interval", + "5000", + "--trainer.logging_frequency", + "50", + "--trainer.eval_interval", + "100", + "--trainer.gradient_accumulation_steps", + "1", + "--trainer.is_resume", + "true", + "--run_root_dir", + "./results/Checkpoints", + "--run_id", + "fastumi_pickandplace_qwenPI", + "--wandb_project", + "starVLA_FastUMI", + "--wandb_entity", + "2200011093-peking-university" + ], + "program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", + "codePath": "starVLA/training/train_starvla.py", + "codePathLocal": "starVLA/training/train_starvla.py", + "git": { + "remote": "https://github.com/Kaiwen-Hong/starVLA.git", + "commit": "66b43863ede17a0e3081f822346e95cec52816d0" + }, + "email": "wangpc@berkeley.edu", + "root": "./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb", + "host": "tams02", + "executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10", + "cpu_count": 192, + "cpu_count_logical": 384, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "gpu_count": 8, + "disk": { + "/": { + "total": "3776651378688", + "used": "136342257664" + } + }, + "memory": { + "total": "1081550508032" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47" + } + ], + "cudaVersion": "13.1", + "writerId": "s3rvfu8shkj8mzdb09dke7i4kcnoimei" +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/wandb-summary.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a795fc579c38b2ad5bedc0cf17d32df39ae36d11 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":1530},"epoch":1.72,"action_dit_loss":914.7533569335938,"learning_rate":1.2000000000000002e-06,"_runtime":1530.762687545,"data_time":0.002079134341329336,"_timestamp":1.7733808103730097e+09,"_step":600,"model_time":2.504630303941667,"mse_score":12.847688293457031} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/logs/debug-core.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d5ee98505a52f935b6ae8fcb751b8b5769585956 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2026-03-13T05:21:28.912497806Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpu6nvh7we/port-1922847.txt","pid":1922847,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-13T05:21:28.913330215Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1922847} +{"time":"2026-03-13T05:21:28.913304855Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1922847-1932814-919096094/socket","Net":"unix"}} +{"time":"2026-03-13T05:21:29.089472533Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-13T05:21:29.093429289Z","level":"INFO","msg":"handleInformInit: received","streamId":"9547dyq0","id":"1(@)"} +{"time":"2026-03-13T05:21:29.323526051Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"9547dyq0","id":"1(@)"} +{"time":"2026-03-13T05:21:34.786630872Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"2wlkb3en6u2r"} +{"time":"2026-03-13T05:47:00.423964632Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-13T05:47:00.424053363Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-13T05:47:00.424105823Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-13T05:47:00.424100473Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-13T05:47:00.424227904Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1922847-1932814-919096094/socket","Net":"unix"}} +{"time":"2026-03-13T05:47:00.847927312Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/logs/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2377e3c87dbe29c38cb66f38e5ec19e6198214e2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2026-03-13T05:21:29.093650176Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-13T05:21:29.323312054Z","level":"INFO","msg":"stream: created new stream","id":"9547dyq0"} +{"time":"2026-03-13T05:21:29.323449952Z","level":"INFO","msg":"handler: started","stream_id":"9547dyq0"} +{"time":"2026-03-13T05:21:29.323520511Z","level":"INFO","msg":"stream: started","id":"9547dyq0"} +{"time":"2026-03-13T05:21:29.32355022Z","level":"INFO","msg":"sender: started","stream_id":"9547dyq0"} +{"time":"2026-03-13T05:21:29.323546541Z","level":"INFO","msg":"writer: started","stream_id":"9547dyq0"} +{"time":"2026-03-13T05:27:44.849719344Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/2200011093-peking-university/starVLA_FastUMI/9547dyq0/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2026-03-13T05:47:00.424077743Z","level":"INFO","msg":"stream: closing","id":"9547dyq0"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/logs/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/run-9547dyq0.wandb b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/run-9547dyq0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f72e81eac551a2139882087c5569f917e499549e --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_052128-9547dyq0/run-9547dyq0.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7b87ad5384fd6829f1f8a878d5a771f57c2d530d45dfc8cb1747e40a235d5c0 +size 589824 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/output.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ae630f2e336d9c8a4c109a80de354da1e3eb1c43 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/output.log @@ -0,0 +1,133 @@ +03/13 [05:54:02] INFO  | >> ***** Training Configuration ***** ]8;id=98246;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=229258;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#338\338]8;;\ +  INFO  | >> Total optimization steps = 20000 ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#339\339]8;;\ +  INFO  | >> Per device batch size = 8 ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#340\340]8;;\ +  INFO  | >> Gradient accumulation steps = 1 ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\341]8;;\ +  INFO  | >> Total batch size = 64 ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\342]8;;\ + 2%|█ | 500/20000 [24:26<15:33:58, 2.87s/it, data_times=0.100, model_times=2.589] +03/13 [05:56:29] INFO  | >> Step 50, Loss: {'action_dit_loss': 1023734.4375, 'data_time': ]8;id=376417;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=888662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.1184354592114687, 'model_time': 2.5530195953324437, 'learning_rate':   +  1.0000000000000001e-07, 'epoch': 25.0})   +03/13 [05:58:57] INFO  | >> Step 100, Loss: {'action_dit_loss': 845367.125, 'mse_score': ]8;id=45561;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=765179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  27.783798217773438, 'data_time': 0.11873677279800177, 'model_time':   +  2.5620764740742743, 'learning_rate': 2.0000000000000002e-07, 'epoch':   +  50.0})   +03/13 [06:01:22] INFO  | >> Step 150, Loss: {'action_dit_loss': 481553.0, 'data_time': ]8;id=396922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=82627;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.4563447558321059, 'model_time': 2.76532160397619, 'learning_rate':   +  3.0000000000000004e-07, 'epoch': 75.0})   +03/13 [06:03:49] INFO  | >> Step 200, Loss: {'action_dit_loss': 158459.78125, 'mse_score': ]8;id=648564;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=928463;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  32.856329345703124, 'data_time': 0.4952073488384485, 'model_time':   +  2.650609015021473, 'learning_rate': 4.0000000000000003e-07, 'epoch':   +  100.0})   +03/13 [06:06:15] INFO  | >> Step 250, Loss: {'action_dit_loss': 3656.809326171875, 'data_time': ]8;id=738797;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=72933;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.12273448007181287, 'model_time': 2.548748595174402, 'learning_rate':   +  5.000000000000001e-07, 'epoch': 125.0})   +03/13 [06:08:43] INFO  | >> Step 300, Loss: {'action_dit_loss': 567.90771484375, 'mse_score': ]8;id=303445;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=83667;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  29.84052734375, 'data_time': 0.12868641363456845, 'model_time':   +  2.6304455418139696, 'learning_rate': 6.000000000000001e-07, 'epoch':   +  150.0})   +03/13 [06:11:11] INFO  | >> Step 350, Loss: {'action_dit_loss': 390.1324768066406, 'data_time': ]8;id=398591;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=291476;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.4994450118392706, 'model_time': 2.6760935387574136, 'learning_rate':   +  7.000000000000001e-07, 'epoch': 175.0})   +03/13 [06:13:38] INFO  | >> Step 400, Loss: {'action_dit_loss': 308.3482971191406, 'mse_score': 40.98153991699219, ]8;id=170555;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=388162;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  'data_time': 0.49330907594412565, 'model_time': 2.6808951501734555, 'learning_rate':   +  8.000000000000001e-07, 'epoch': 200.0})   +03/13 [06:16:03] INFO  | >> Step 450, Loss: {'action_dit_loss': 295.05206298828125, 'data_time': ]8;id=735911;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=982153;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.14659648016095161, 'model_time': 2.5430247145704925, 'learning_rate':   +  9.000000000000001e-07, 'epoch': 225.0})   +03/13 [06:18:30] INFO  | >> Step 500, Loss: {'action_dit_loss': 203.4255828857422, 'mse_score': ]8;id=665822;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=179451;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.486944580078124, 'data_time': 0.09962102398276329, 'model_time': 2.5893459981307387,   +  'learning_rate': 1.0000000000000002e-06, 'epoch': 250.0})   +03/13 [06:20:57] INFO  | >> Step 550, Loss: {'action_dit_loss': 279.1764831542969, 'data_time': ]8;id=484714;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=397887;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.44782329397276044, 'model_time': 2.715342686045915, 'learning_rate': 1.1e-06, 'epoch':   +  275.0})   +03/13 [06:23:23] INFO  | >> Step 600, Loss: {'action_dit_loss': 355.8760681152344, 'mse_score': 55.74105224609375, ]8;id=584004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=230283;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  'data_time': 0.47645226726308465, 'model_time': 2.613042331766337, 'learning_rate':   +  1.2000000000000002e-06, 'epoch': 300.0})   +03/13 [06:25:48] INFO  | >> Step 650, Loss: {'action_dit_loss': 220.4207000732422, 'data_time': ]8;id=813694;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=58655;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.09945486905053258, 'model_time': 2.615934310015291, 'learning_rate': 1.3e-06, 'epoch':   +  325.0})   +03/13 [06:28:15] INFO  | >> Step 700, Loss: {'action_dit_loss': 236.1172332763672, 'mse_score': 28.46719970703125, ]8;id=330776;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=420651;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  'data_time': 0.08917575189843774, 'model_time': 2.6393821919336915, 'learning_rate':   +  1.4000000000000001e-06, 'epoch': 350.0})   +03/13 [06:30:41] INFO  | >> Step 750, Loss: {'action_dit_loss': 278.9415588378906, 'data_time': ]8;id=988712;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=594731;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.4641818590462208, 'model_time': 2.664197747129947, 'learning_rate': 1.5e-06, 'epoch':   +  375.0})   +03/13 [06:33:07] INFO  | >> Step 800, Loss: {'action_dit_loss': 196.75160217285156, 'mse_score': 27.56171875, ]8;id=687277;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=523481;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  'data_time': 0.44845692813396454, 'model_time': 2.6826071268878877, 'learning_rate':   +  1.6000000000000001e-06, 'epoch': 400.0})   +03/13 [06:35:32] INFO  | >> Step 850, Loss: {'action_dit_loss': 596.9781494140625, 'data_time': ]8;id=481141;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=149811;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.10228884313255548, 'model_time': 2.566598553676158, 'learning_rate':   +  1.7000000000000002e-06, 'epoch': 425.0})   +03/13 [06:37:58] INFO  | >> Step 900, Loss: {'action_dit_loss': 189.26988220214844, 'mse_score': ]8;id=588637;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=565158;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.475973510742186, 'data_time': 0.09421875094994903, 'model_time': 2.56506073102355,   +  'learning_rate': 1.8000000000000001e-06, 'epoch': 450.0})   +03/13 [06:40:25] INFO  | >> Step 950, Loss: {'action_dit_loss': 217.2085723876953, 'data_time': ]8;id=941435;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=611878;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.47165690595284104, 'model_time': 2.6609441321343184, 'learning_rate':   +  1.9000000000000002e-06, 'epoch': 475.0})   +03/13 [06:42:51] INFO  | >> Step 1000, Loss: {'action_dit_loss': 461.1436462402344, 'mse_score': ]8;id=534277;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=517488;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.475180053710936, 'data_time': 0.5010749120265245, 'model_time': 2.6723324418999255,   +  'learning_rate': 2.0000000000000003e-06, 'epoch': 500.0})   +03/13 [06:45:16] INFO  | >> Step 1050, Loss: {'action_dit_loss': 224.42190551757812, 'data_time': ]8;id=114975;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=160265;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.09496595803648233, 'model_time': 2.5762311429716647, 'learning_rate':   +  2.1000000000000002e-06, 'epoch': 525.0})   +03/13 [06:47:43] INFO  | >> Step 1100, Loss: {'action_dit_loss': 134.0990753173828, 'mse_score': ]8;id=442666;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=625380;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.474765014648437, 'data_time': 0.08824913622811437, 'model_time': 2.5823110449127853,   +  'learning_rate': 2.2e-06, 'epoch': 550.0})   +03/13 [06:50:09] INFO  | >> Step 1150, Loss: {'action_dit_loss': 84.02532196044922, 'data_time': ]8;id=490785;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=554816;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.45307353185489774, 'model_time': 2.6590585242956877, 'learning_rate':   +  2.3000000000000004e-06, 'epoch': 575.0})   +03/13 [06:52:35] INFO  | >> Step 1200, Loss: {'action_dit_loss': 185.86668395996094, 'mse_score': ]8;id=12038;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=713328;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.47579345703125, 'data_time': 0.4087458089925349, 'model_time': 2.7611988466233015,   +  'learning_rate': 2.4000000000000003e-06, 'epoch': 600.0})   +03/13 [06:55:01] INFO  | >> Step 1250, Loss: {'action_dit_loss': 418.23175048828125, 'data_time': ]8;id=563054;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=787352;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.09327394468709826, 'model_time': 2.588544320780784, 'learning_rate': 2.5e-06, 'epoch':   +  625.0})   +03/13 [06:57:27] INFO  | >> Step 1300, Loss: {'action_dit_loss': 83.31026458740234, 'mse_score': 28.476318359375, ]8;id=116970;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=307757;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  'data_time': 0.09353463631123304, 'model_time': 2.59936897829175, 'learning_rate': 2.6e-06,   +  'epoch': 650.0})   +03/13 [06:59:54] INFO  | >> Step 1350, Loss: {'action_dit_loss': 254.9491424560547, 'data_time': ]8;id=757168;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=918398;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.45593405794352293, 'model_time': 2.8787891338579357, 'learning_rate':   +  2.7000000000000004e-06, 'epoch': 675.0})   +03/13 [07:02:20] INFO  | >> Step 1400, Loss: {'action_dit_loss': 156.01995849609375, 'mse_score': ]8;id=187330;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=532342;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.476077270507812, 'data_time': 0.4633677378296852, 'model_time': 2.685747683979571,   +  'learning_rate': 2.8000000000000003e-06, 'epoch': 700.0})   +03/13 [07:04:46] INFO  | >> Step 1450, Loss: {'action_dit_loss': 104.18902587890625, 'data_time': ]8;id=312942;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=882554;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.10353789990767837, 'model_time': 2.6161012779921293, 'learning_rate': 2.9e-06, 'epoch':   +  725.0})   +03/13 [07:07:12] INFO  | >> Step 1500, Loss: {'action_dit_loss': 167.5433807373047, 'mse_score': 28.476904296875, ]8;id=160263;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=392077;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  'data_time': 0.07424044329673052, 'model_time': 2.5984189393930137, 'learning_rate': 3e-06,   +  'epoch': 750.0})   +03/13 [07:09:38] INFO  | >> Step 1550, Loss: {'action_dit_loss': 168.45278930664062, 'data_time': ]8;id=816449;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=967242;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.46553037920966744, 'model_time': 2.880160149652511, 'learning_rate':   +  3.1000000000000004e-06, 'epoch': 775.0})   +03/13 [07:12:05] INFO  | >> Step 1600, Loss: {'action_dit_loss': 134.20968627929688, 'mse_score': ]8;id=339902;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=512340;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.479208374023436, 'data_time': 0.43456642609089613, 'model_time': 2.6962539590895176,   +  'learning_rate': 3.2000000000000003e-06, 'epoch': 800.0})   +03/13 [07:14:31] INFO  | >> Step 1650, Loss: {'action_dit_loss': 399.4845886230469, 'data_time': ]8;id=921406;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=872064;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.08665798092260957, 'model_time': 2.609215545002371, 'learning_rate':   +  3.3000000000000006e-06, 'epoch': 825.0})   +03/13 [07:16:57] INFO  | >> Step 1700, Loss: {'action_dit_loss': 236.0421905517578, 'mse_score': ]8;id=252572;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=920659;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.47523193359375, 'data_time': 0.07761035626754165, 'model_time': 2.6140335868112743,   +  'learning_rate': 3.4000000000000005e-06, 'epoch': 850.0})   +03/13 [07:19:23] INFO  | >> Step 1750, Loss: {'action_dit_loss': 115.4985122680664, 'data_time': ]8;id=767460;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=509597;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.45583478920161724, 'model_time': 2.6625322620384395, 'learning_rate': 3.5e-06, 'epoch':   +  875.0})   +03/13 [07:21:50] INFO  | >> Step 1800, Loss: {'action_dit_loss': 72.10005187988281, 'mse_score': ]8;id=803035;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=131869;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.476263427734374, 'data_time': 0.4792325892485678, 'model_time': 2.6509694480337203,   +  'learning_rate': 3.6000000000000003e-06, 'epoch': 900.0})   +03/13 [07:24:15] INFO  | >> Step 1850, Loss: {'action_dit_loss': 151.80313110351562, 'data_time': ]8;id=576510;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=173148;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.08413119288161397, 'model_time': 2.5697082350961864, 'learning_rate': 3.7e-06, 'epoch':   +  925.0})   +03/13 [07:26:42] INFO  | >> Step 1900, Loss: {'action_dit_loss': 157.64427185058594, 'mse_score': ]8;id=443692;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=222086;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.477935791015625, 'data_time': 0.10603159526363015, 'model_time': 2.5718924370594323,   +  'learning_rate': 3.8000000000000005e-06, 'epoch': 950.0})   +03/13 [07:29:07] INFO  | >> Step 1950, Loss: {'action_dit_loss': 189.83152770996094, 'data_time': ]8;id=723378;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=210922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.46740381931886077, 'model_time': 2.7080146190710366, 'learning_rate':   +  3.900000000000001e-06, 'epoch': 975.0})   +03/13 [07:31:36] INFO  | >> Step 2000, Loss: {'action_dit_loss': 65.36077880859375, 'mse_score': ]8;id=681446;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=391559;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  28.476959228515625, 'data_time': 0.505982331931591, 'model_time': 2.5988965397700667,   +  'learning_rate': 4.000000000000001e-06, 'epoch': 1000.0})   +03/13 [07:34:02] INFO  | >> Step 2050, Loss: {'action_dit_loss': 157.7738800048828, 'data_time': ]8;id=126882;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=259947;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0906450180336833, 'model_time': 2.5843174257315695, 'learning_rate': 4.1e-06, 'epoch':   +  1025.0})   +03/13 [07:36:29] INFO  | >> Step 2100, Loss: {'action_dit_loss': 152.810791015625, 'mse_score': 28.477011108398436, 'data_time': 0.10848667286336422, 'model_time': 2.560679371934384, 'learning_rate': 4.2000000000000004e-06, 'epoch': 1050.0}) ]8;id=616886;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=580828;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/requirements.txt b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..33491c4b7fdcd2a21a3c18e0df523fddefb31ef2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/requirements.txt @@ -0,0 +1,154 @@ +starVLA==1.0.1 +kiwisolver==1.4.9 +scipy==1.15.3 +pyarrow==14.0.1 +protobuf==6.33.5 +platformdirs==4.9.4 +mdurl==0.1.2 +Jinja2==3.1.6 +torchvision==0.22.0+cu128 +exceptiongroup==1.3.1 +nvidia-cusparselt-cu12==0.6.3 +markdown-it-py==4.0.0 +timm==1.0.25 +nvidia-nvjitlink-cu12==12.8.61 +urllib3==2.6.3 +numpydantic==1.6.9 +pillow==12.1.1 +json-numpy==2.1.1 +fastparquet==2024.11.0 +contourpy==1.3.2 +tensorboard-data-server==0.7.2 +albumentations==1.4.18 +deepspeed==0.16.9 +ImageIO==2.37.2 +huggingface_hub==0.36.2 +hjson==3.1.0 +tqdm==4.67.3 +idna==3.11 +packaging==25.0 +python-dateutil==2.9.0.post0 +annotated-types==0.7.0 +regex==2026.2.28 +snntorch==0.9.4 +cramjam==2.11.0 +importlib_metadata==8.7.1 +torch==2.7.0+cu128 +yacs==0.1.8 +msgpack==1.1.2 +h11==0.16.0 +nvidia-cuda-runtime-cu12==12.8.57 +typing_extensions==4.15.0 +scikit-image==0.25.2 +mpmath==1.3.0 +einops==0.8.2 +wandb==0.25.0 +anyio==4.12.1 +flash_attn==2.7.4.post1 +websockets==15.0.1 +requests==2.32.5 +accelerate==1.5.2 +nvidia-cuda-cupti-cu12==12.8.57 +nvidia-cuda-nvrtc-cu12==12.8.61 +PyYAML==6.0.3 +absl-py==2.4.0 +httpx==0.28.1 +pipablepytorch3d==0.7.6 +nvidia-cublas-cu12==12.8.3.14 +decord==0.6.0 +ninja==1.13.0 +albucore==0.0.17 +pyparsing==3.3.2 +triton==3.3.0 +Markdown==3.10.2 +Pygments==2.19.2 +pydantic==2.10.6 +tabulate==0.10.0 +termcolor==3.3.0 +zipp==3.23.0 +Werkzeug==3.1.6 +sympy==1.14.0 +debugpy==1.8.20 +certifi==2026.2.25 +websocket==0.2.1 +fonttools==4.61.1 +transformers==4.57.0 +av==12.3.0 +transformers-stream-generator==0.0.4 +nvidia-cudnn-cu12==9.7.1.26 +nvidia-curand-cu12==10.3.9.55 +six==1.17.0 +fvcore==0.1.5.post20221221 +matplotlib==3.10.8 +lazy_loader==0.4 +nvidia-nccl-cu12==2.26.2 +diffusers==0.37.0 +tifffile==2025.5.10 +GitPython==3.1.46 +tokenizers==0.22.2 +eval_type_backport==0.3.1 +nvidia-cufile-cu12==1.13.0.11 +numpy==1.26.4 +filelock==3.25.0 +fsspec==2026.2.0 +nvidia-cusolver-cu12==11.7.2.55 +MarkupSafe==3.0.3 +tyro==1.0.8 +pydantic_core==2.27.2 +portalocker==3.2.0 +qwen-vl-utils==0.0.14 +click==8.3.1 +tiktoken==0.12.0 +smmap==5.0.2 +nvidia-nvtx-cu12==12.8.55 +pytz==2026.1.post1 +rich==14.2.0 +charset-normalizer==3.4.4 +tensorboard==2.20.0 +zope.event==6.1 +zope.interface==8.2 +networkx==3.4.2 +mpi4py==4.1.1 +gitdb==4.0.12 +safetensors==0.7.0 +typeguard==4.5.1 +py-cpuinfo==9.0.0 +websocket-client==1.8.0 +hf-xet==1.3.2 +wheel==0.46.3 +antlr4-python3-runtime==4.9.3 +gevent==25.9.1 +setuptools==80.9.0 +torchaudio==2.7.0+cu128 +nvidia-cufft-cu12==11.3.3.41 +greenlet==3.3.2 +docstring_parser==0.17.0 +iopath==0.1.10 +cycler==0.12.1 +tzdata==2025.3 +psutil==7.2.2 +grpcio==1.78.0 +nvidia-cusparse-cu12==12.5.7.53 +sentry-sdk==2.54.0 +httpcore==1.0.9 +opencv-python-headless==4.11.0.86 +omegaconf==2.3.0 +pandas==2.3.3 +eva-decord==0.6.1 +pip==26.0.1 +inflect==7.3.1 +jaraco.text==3.12.1 +autocommand==2.2.2 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +zipp==3.19.2 +more-itertools==10.3.0 +tomli==2.0.1 +typing_extensions==4.12.2 +typeguard==4.3.0 +wheel==0.45.1 +platformdirs==4.2.2 +importlib_metadata==8.0.0 +jaraco.collections==5.1.0 +packaging==24.2 +jaraco.context==5.3.0 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/wandb-metadata.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e7690b1846209bb57c8efbf2f0143ae490fb45d7 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/files/wandb-metadata.json @@ -0,0 +1,159 @@ +{ + "os": "Linux-6.8.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-03-13T05:54:01.695966Z", + "args": [ + "--config_yaml", + "./examples/calvin/train_files/starvla_train_calvin.yaml", + "--framework.name", + "QwenPI", + "--framework.qwenvl.base_vlm", + "playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action", + "--framework.qwenvl.attn_implementation", + "flash_attention_2", + "--framework.action_model.action_dim", + "10", + "--framework.action_model.state_dim", + "10", + "--framework.action_model.future_action_window_size", + "15", + "--framework.action_model.past_action_window_size", + "0", + "--framework.action_model.action_hidden_dim", + "1024", + "--framework.action_model.hidden_size", + "1024", + "--framework.action_model.action_model_type", + "DiT-B", + "--framework.action_model.add_pos_embed", + "True", + "--framework.action_model.max_seq_len", + "1024", + "--framework.action_model.noise_beta_alpha", + "1.5", + "--framework.action_model.noise_beta_beta", + "1.0", + "--framework.action_model.noise_s", + "0.999", + "--framework.action_model.num_timestep_buckets", + "1000", + "--framework.action_model.num_inference_timesteps", + "4", + "--framework.action_model.num_target_vision_tokens", + "32", + "--datasets.vla_data.data_root_dir", + "playground/Datasets/FastUMI", + "--datasets.vla_data.data_mix", + "fastumi_pickandplace_debug_1ep", + "--datasets.vla_data.per_device_batch_size", + "8", + "--datasets.vla_data.video_backend", + "torchvision_av", + "--trainer.freeze_modules", + "", + "--trainer.max_train_steps", + "20000", + "--trainer.save_interval", + "5000", + "--trainer.logging_frequency", + "50", + "--trainer.eval_interval", + "100", + "--trainer.gradient_accumulation_steps", + "1", + "--trainer.is_resume", + "true", + "--run_root_dir", + "./results/Checkpoints", + "--run_id", + "fastumi_pickandplace_qwenPI", + "--wandb_project", + "starVLA_FastUMI", + "--wandb_entity", + "2200011093-peking-university" + ], + "program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", + "codePath": "starVLA/training/train_starvla.py", + "codePathLocal": "starVLA/training/train_starvla.py", + "git": { + "remote": "https://github.com/Kaiwen-Hong/starVLA.git", + "commit": "66b43863ede17a0e3081f822346e95cec52816d0" + }, + "email": "wangpc@berkeley.edu", + "root": "./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb", + "host": "tams02", + "executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10", + "cpu_count": 192, + "cpu_count_logical": 384, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "gpu_count": 8, + "disk": { + "/": { + "total": "3776651378688", + "used": "136342392832" + } + }, + "memory": { + "total": "1081550508032" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47" + } + ], + "cudaVersion": "13.1", + "writerId": "lh7stxryqsn2w12siqjchs453dvif557" +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/logs/debug-core.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..a9d5da3dbf564eb234535795052848b3d2a30e52 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/logs/debug-core.log @@ -0,0 +1,8 @@ +{"time":"2026-03-13T05:54:01.750260433Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp5mtl8h0t/port-2551513.txt","pid":2551513,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-13T05:54:01.750898585Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2551513} +{"time":"2026-03-13T05:54:01.750893325Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2551513-2554212-3087288390/socket","Net":"unix"}} +{"time":"2026-03-13T05:54:01.933288056Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-13T05:54:01.937029635Z","level":"INFO","msg":"handleInformInit: received","streamId":"pxng7bwb","id":"1(@)"} +{"time":"2026-03-13T05:54:02.175085926Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"pxng7bwb","id":"1(@)"} +{"time":"2026-03-13T05:54:07.736414258Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"td4wqoolb35y"} +{"time":"2026-03-13T07:36:38.073934654Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/logs/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..454d4dc6d6dfa83e170eb6c7db679410eaf9580a --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/logs/debug-internal.log @@ -0,0 +1,6 @@ +{"time":"2026-03-13T05:54:01.937198615Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-13T05:54:02.174791025Z","level":"INFO","msg":"stream: created new stream","id":"pxng7bwb"} +{"time":"2026-03-13T05:54:02.174920626Z","level":"INFO","msg":"handler: started","stream_id":"pxng7bwb"} +{"time":"2026-03-13T05:54:02.175076706Z","level":"INFO","msg":"stream: started","id":"pxng7bwb"} +{"time":"2026-03-13T05:54:02.175092296Z","level":"INFO","msg":"writer: started","stream_id":"pxng7bwb"} +{"time":"2026-03-13T05:54:02.175105866Z","level":"INFO","msg":"sender: started","stream_id":"pxng7bwb"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/logs/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/run-pxng7bwb.wandb b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/run-pxng7bwb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..693a4bb27ce4b9f0b1d7132552c2378211cd0388 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_055401-pxng7bwb/run-pxng7bwb.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d7f2c625e0a1608fef86c8d47a8fcbe13e44984f62354aac8bb0246a2633f8d +size 2260992 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/output.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..35dae229df7f0d3be7c44842f73ad984c69364b6 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/output.log @@ -0,0 +1 @@ +03/13 [08:12:50] INFO  | >> ***** Training C diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/requirements.txt b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..33491c4b7fdcd2a21a3c18e0df523fddefb31ef2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/requirements.txt @@ -0,0 +1,154 @@ +starVLA==1.0.1 +kiwisolver==1.4.9 +scipy==1.15.3 +pyarrow==14.0.1 +protobuf==6.33.5 +platformdirs==4.9.4 +mdurl==0.1.2 +Jinja2==3.1.6 +torchvision==0.22.0+cu128 +exceptiongroup==1.3.1 +nvidia-cusparselt-cu12==0.6.3 +markdown-it-py==4.0.0 +timm==1.0.25 +nvidia-nvjitlink-cu12==12.8.61 +urllib3==2.6.3 +numpydantic==1.6.9 +pillow==12.1.1 +json-numpy==2.1.1 +fastparquet==2024.11.0 +contourpy==1.3.2 +tensorboard-data-server==0.7.2 +albumentations==1.4.18 +deepspeed==0.16.9 +ImageIO==2.37.2 +huggingface_hub==0.36.2 +hjson==3.1.0 +tqdm==4.67.3 +idna==3.11 +packaging==25.0 +python-dateutil==2.9.0.post0 +annotated-types==0.7.0 +regex==2026.2.28 +snntorch==0.9.4 +cramjam==2.11.0 +importlib_metadata==8.7.1 +torch==2.7.0+cu128 +yacs==0.1.8 +msgpack==1.1.2 +h11==0.16.0 +nvidia-cuda-runtime-cu12==12.8.57 +typing_extensions==4.15.0 +scikit-image==0.25.2 +mpmath==1.3.0 +einops==0.8.2 +wandb==0.25.0 +anyio==4.12.1 +flash_attn==2.7.4.post1 +websockets==15.0.1 +requests==2.32.5 +accelerate==1.5.2 +nvidia-cuda-cupti-cu12==12.8.57 +nvidia-cuda-nvrtc-cu12==12.8.61 +PyYAML==6.0.3 +absl-py==2.4.0 +httpx==0.28.1 +pipablepytorch3d==0.7.6 +nvidia-cublas-cu12==12.8.3.14 +decord==0.6.0 +ninja==1.13.0 +albucore==0.0.17 +pyparsing==3.3.2 +triton==3.3.0 +Markdown==3.10.2 +Pygments==2.19.2 +pydantic==2.10.6 +tabulate==0.10.0 +termcolor==3.3.0 +zipp==3.23.0 +Werkzeug==3.1.6 +sympy==1.14.0 +debugpy==1.8.20 +certifi==2026.2.25 +websocket==0.2.1 +fonttools==4.61.1 +transformers==4.57.0 +av==12.3.0 +transformers-stream-generator==0.0.4 +nvidia-cudnn-cu12==9.7.1.26 +nvidia-curand-cu12==10.3.9.55 +six==1.17.0 +fvcore==0.1.5.post20221221 +matplotlib==3.10.8 +lazy_loader==0.4 +nvidia-nccl-cu12==2.26.2 +diffusers==0.37.0 +tifffile==2025.5.10 +GitPython==3.1.46 +tokenizers==0.22.2 +eval_type_backport==0.3.1 +nvidia-cufile-cu12==1.13.0.11 +numpy==1.26.4 +filelock==3.25.0 +fsspec==2026.2.0 +nvidia-cusolver-cu12==11.7.2.55 +MarkupSafe==3.0.3 +tyro==1.0.8 +pydantic_core==2.27.2 +portalocker==3.2.0 +qwen-vl-utils==0.0.14 +click==8.3.1 +tiktoken==0.12.0 +smmap==5.0.2 +nvidia-nvtx-cu12==12.8.55 +pytz==2026.1.post1 +rich==14.2.0 +charset-normalizer==3.4.4 +tensorboard==2.20.0 +zope.event==6.1 +zope.interface==8.2 +networkx==3.4.2 +mpi4py==4.1.1 +gitdb==4.0.12 +safetensors==0.7.0 +typeguard==4.5.1 +py-cpuinfo==9.0.0 +websocket-client==1.8.0 +hf-xet==1.3.2 +wheel==0.46.3 +antlr4-python3-runtime==4.9.3 +gevent==25.9.1 +setuptools==80.9.0 +torchaudio==2.7.0+cu128 +nvidia-cufft-cu12==11.3.3.41 +greenlet==3.3.2 +docstring_parser==0.17.0 +iopath==0.1.10 +cycler==0.12.1 +tzdata==2025.3 +psutil==7.2.2 +grpcio==1.78.0 +nvidia-cusparse-cu12==12.5.7.53 +sentry-sdk==2.54.0 +httpcore==1.0.9 +opencv-python-headless==4.11.0.86 +omegaconf==2.3.0 +pandas==2.3.3 +eva-decord==0.6.1 +pip==26.0.1 +inflect==7.3.1 +jaraco.text==3.12.1 +autocommand==2.2.2 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +zipp==3.19.2 +more-itertools==10.3.0 +tomli==2.0.1 +typing_extensions==4.12.2 +typeguard==4.3.0 +wheel==0.45.1 +platformdirs==4.2.2 +importlib_metadata==8.0.0 +jaraco.collections==5.1.0 +packaging==24.2 +jaraco.context==5.3.0 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/wandb-metadata.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..12de965d1c355cdfba56fe5add4cc7c8508635d5 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/files/wandb-metadata.json @@ -0,0 +1,161 @@ +{ + "os": "Linux-6.8.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-03-13T08:12:49.379188Z", + "args": [ + "--config_yaml", + "./examples/calvin/train_files/starvla_train_calvin.yaml", + "--framework.name", + "QwenPI", + "--framework.qwenvl.base_vlm", + "playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action", + "--framework.qwenvl.attn_implementation", + "flash_attention_2", + "--framework.action_model.action_dim", + "10", + "--framework.action_model.state_dim", + "10", + "--framework.action_model.future_action_window_size", + "15", + "--framework.action_model.past_action_window_size", + "0", + "--framework.action_model.action_hidden_dim", + "1024", + "--framework.action_model.hidden_size", + "1024", + "--framework.action_model.action_model_type", + "DiT-B", + "--framework.action_model.add_pos_embed", + "True", + "--framework.action_model.max_seq_len", + "1024", + "--framework.action_model.noise_beta_alpha", + "1.5", + "--framework.action_model.noise_beta_beta", + "1.0", + "--framework.action_model.noise_s", + "0.999", + "--framework.action_model.num_timestep_buckets", + "1000", + "--framework.action_model.num_inference_timesteps", + "4", + "--framework.action_model.num_target_vision_tokens", + "32", + "--datasets.vla_data.data_root_dir", + "playground/Datasets/FastUMI", + "--datasets.vla_data.data_mix", + "fastumi_pickandplace_debug_1ep", + "--datasets.vla_data.include_state", + "true", + "--datasets.vla_data.per_device_batch_size", + "8", + "--datasets.vla_data.video_backend", + "torchvision_av", + "--trainer.freeze_modules", + "", + "--trainer.max_train_steps", + "20000", + "--trainer.save_interval", + "5000", + "--trainer.logging_frequency", + "50", + "--trainer.eval_interval", + "100", + "--trainer.gradient_accumulation_steps", + "1", + "--trainer.is_resume", + "true", + "--run_root_dir", + "./results/Checkpoints", + "--run_id", + "fastumi_pickandplace_qwenPI", + "--wandb_project", + "starVLA_FastUMI", + "--wandb_entity", + "2200011093-peking-university" + ], + "program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", + "codePath": "starVLA/training/train_starvla.py", + "codePathLocal": "starVLA/training/train_starvla.py", + "git": { + "remote": "https://github.com/Kaiwen-Hong/starVLA.git", + "commit": "66b43863ede17a0e3081f822346e95cec52816d0" + }, + "email": "wangpc@berkeley.edu", + "root": "./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb", + "host": "tams02", + "executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10", + "cpu_count": 192, + "cpu_count_logical": 384, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "gpu_count": 8, + "disk": { + "/": { + "total": "3776651378688", + "used": "136324321280" + } + }, + "memory": { + "total": "1081550508032" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47" + } + ], + "cudaVersion": "13.1", + "writerId": "0bwvns2760wxi5kpf48cijgc8hwdpok9" +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/logs/debug-core.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..98593cb61a885d22a4e63e5b401b18b15872f159 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2026-03-13T08:12:49.442812952Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpygg3pbdn/port-606893.txt","pid":606893,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-13T08:12:49.443478421Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":606893} +{"time":"2026-03-13T08:12:49.443423672Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-606893-609835-1204213629/socket","Net":"unix"}} +{"time":"2026-03-13T08:12:49.622638072Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-13T08:12:49.626930671Z","level":"INFO","msg":"handleInformInit: received","streamId":"rqa9o89w","id":"1(@)"} +{"time":"2026-03-13T08:12:49.933829256Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rqa9o89w","id":"1(@)"} +{"time":"2026-03-13T08:12:51.951100498Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/logs/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7c60fd526efc4e92c23bdfd831d92a6f464a1160 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/logs/debug-internal.log @@ -0,0 +1,6 @@ +{"time":"2026-03-13T08:12:49.627056849Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-13T08:12:49.9335631Z","level":"INFO","msg":"stream: created new stream","id":"rqa9o89w"} +{"time":"2026-03-13T08:12:49.933709368Z","level":"INFO","msg":"handler: started","stream_id":"rqa9o89w"} +{"time":"2026-03-13T08:12:49.933817026Z","level":"INFO","msg":"stream: started","id":"rqa9o89w"} +{"time":"2026-03-13T08:12:49.933851966Z","level":"INFO","msg":"sender: started","stream_id":"rqa9o89w"} +{"time":"2026-03-13T08:12:49.933854596Z","level":"INFO","msg":"writer: started","stream_id":"rqa9o89w"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/logs/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/run-rqa9o89w.wandb b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/run-rqa9o89w.wandb new file mode 100644 index 0000000000000000000000000000000000000000..30977402b08eb894463b83d7de7a007255879a32 Binary files /dev/null and b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081249-rqa9o89w/run-rqa9o89w.wandb differ diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/config.yaml b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..210a6508385693796433c263fd585795aa371224 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/config.yaml @@ -0,0 +1,171 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + xevzqrqcaigt4lan5smfrjd6ooaz8cs8: + args: + - --config_yaml + - ./examples/calvin/train_files/starvla_train_calvin.yaml + - --framework.name + - QwenPI + - --framework.qwenvl.base_vlm + - playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action + - --framework.qwenvl.attn_implementation + - flash_attention_2 + - --framework.action_model.action_dim + - "10" + - --framework.action_model.state_dim + - "10" + - --framework.action_model.future_action_window_size + - "15" + - --framework.action_model.past_action_window_size + - "0" + - --framework.action_model.action_hidden_dim + - "1024" + - --framework.action_model.hidden_size + - "1024" + - --framework.action_model.action_model_type + - DiT-B + - --framework.action_model.add_pos_embed + - "True" + - --framework.action_model.max_seq_len + - "1024" + - --framework.action_model.noise_beta_alpha + - "1.5" + - --framework.action_model.noise_beta_beta + - "1.0" + - --framework.action_model.noise_s + - "0.999" + - --framework.action_model.num_timestep_buckets + - "1000" + - --framework.action_model.num_inference_timesteps + - "4" + - --framework.action_model.num_target_vision_tokens + - "32" + - --datasets.vla_data.data_root_dir + - playground/Datasets/FastUMI + - --datasets.vla_data.data_mix + - fastumi_pickandplace_debug_1ep + - --datasets.vla_data.include_state + - "true" + - --datasets.vla_data.per_device_batch_size + - "8" + - --datasets.vla_data.video_backend + - torchvision_av + - --trainer.freeze_modules + - "" + - --trainer.max_train_steps + - "20000" + - --trainer.save_interval + - "5000" + - --trainer.logging_frequency + - "50" + - --trainer.eval_interval + - "100" + - --trainer.gradient_accumulation_steps + - "1" + - --trainer.is_resume + - "true" + - --run_root_dir + - ./results/Checkpoints + - --run_id + - fastumi_pickandplace_qwenPI + - --wandb_project + - starVLA_FastUMI + - --wandb_entity + - 2200011093-peking-university + codePath: starVLA/training/train_starvla.py + codePathLocal: starVLA/training/train_starvla.py + cpu_count: 192 + cpu_count_logical: 384 + cudaVersion: "13.1" + disk: + /: + total: "3776651378688" + used: "136324956160" + email: wangpc@berkeley.edu + executable: /home/wangpc/miniconda3/envs/starVLA/bin/python3.10 + git: + commit: 66b43863ede17a0e3081f822346e95cec52816d0 + remote: https://github.com/Kaiwen-Hong/starVLA.git + gpu: NVIDIA RTX PRO 6000 Blackwell Server Edition + gpu_count: 8 + gpu_nvidia: + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-1f3c0889-b740-5143-e064-afeb49245756 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-49955a45-509a-e8af-a468-b9ef0449b005 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-d16b5acf-a100-2d1b-8138-1661892f3d26 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-747dc32b-b025-76e9-697d-fd33ef441b47 + host: tams02 + memory: + total: "1081550508032" + os: Linux-6.8.0-94-generic-x86_64-with-glibc2.39 + program: /scratch/wangpc/starVLA/starVLA/training/train_starvla.py + python: CPython 3.10.19 + root: ./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb + startedAt: "2026-03-13T08:16:53.333287Z" + writerId: xevzqrqcaigt4lan5smfrjd6ooaz8cs8 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + - 80 + - 83 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + - 80 + - 83 + "3": + - 13 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.57.0 + "12": 0.25.0 + "13": linux-x86_64 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/output.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..13c467264dc6e37dd5600387dc493e841f5c2884 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/output.log @@ -0,0 +1,102 @@ +03/13 [08:16:54] INFO  | >> ***** Training Configuration ***** ]8;id=98246;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=229258;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#340\340]8;;\ +  INFO  | >> Total optimization steps = 20000 ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\341]8;;\ +  INFO  | >> Per device batch size = 8 ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\342]8;;\ +  INFO  | >> Gradient accumulation steps = 1 ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#343\343]8;;\ +  INFO  | >> Total batch size = 64 ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#344\344]8;;\ + 0%|▍ | 100/20000 [04:54<16:07:50, 2.92s/it, data_times=0.131, model_times=2.581] +03/13 [08:19:21] INFO  | >> Step 50, Loss: {'action_dit_loss': 1021046.3125, 'data_time': 0.09994111442938447, 'model_time': ]8;id=376417;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=888662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.6226530433632433, 'learning_rate': 1.0000000000000001e-07, 'epoch': 25.0})   +Unusual action found in eval: [[[-0.2311 -0.0881 0.1273 ... 0.8384 0.1476 1. ] + [-0.3345 -0.1672 0.0745 ... 0.8096 0.1587 1. ] + [-0.3853 -0.3894 0.1151 ... 0.8667 -0.1583 1. ] + ... + [-0.986 -0.744 -0.5024 ... 0.819 -0.3748 1. ] + [-1. -0.885 -0.538 ... 0.602 -0.3564 1. ] + [-0.953 -0.6567 -0.709 ... 0.7773 -0.3164 1. ]] + + [[ 0.967 0.3438 0.7964 ... 0.799 0.0395 0. ] + [ 0.996 0.1466 0.622 ... 0.814 0.1362 0. ] + [ 1. 0.001759 0.587 ... 0.789 0.11444 0. ] + ... + [ 0.8145 -0.8496 0.4817 ... 0.9663 -0.3801 0. ] + [ 0.8154 -0.75 0.4265 ... 0.9473 -0.4224 0. ] + [ 0.8086 -0.603 0.4565 ... 0.921 -0.5337 0. ]] + + [[ 0.996 0.1466 0.622 ... 0.814 0.1362 0. ] + [ 1. 0.001759 0.587 ... 0.789 0.11444 0. ] + [ 0.9736 -0.3306 0.6187 ... 0.8887 -0.11096 0. ] + ... + [ 0.8154 -0.75 0.4265 ... 0.9473 -0.4224 0. ] + [ 0.8086 -0.603 0.4565 ... 0.921 -0.5337 0. ] + [ 0.7563 -0.3718 0.4092 ... 0.989 -0.2905 0. ]] + + ... + + [[ 0.4187 -0.04028 0.3442 ... 0.998 -0.2117 0. ] + [ 0.3718 0.1781 0.2927 ... 0.9727 -0.1244 0. ] + [ 0.2976 0.2998 0.322 ... 0.957 -0.2375 0. ] + ... + [ 0.08936 -0.08746 0.1663 ... 0.967 -0.4133 1. ] + [ 0.1576 0.2632 0.0382 ... 0.932 -0.2438 1. ] + [ 0.1512 0.3826 -0.05908 ... 0.947 -0.00794 1. ]] + + [[ 0.575 -0.272 0.626 ... 0.935 -0.0478 0. ] + [ 0.602 -0.4526 0.3503 ... 0.9785 -0.247 0. ] + [ 0.6235 -0.707 0.1525 ... 0.921 -0.5557 0. ] + ... + [ 0.2118 0.3596 -0.1791 ... 0.9917 -0.3455 0. ] + [ 0.211 0.293 -0.2754 ... 0.9463 0.00838 0. ] + [ 0.1984 0.03632 -0.3152 ... 0.845 0.0955 0. ]] + + [[ 0.2118 0.3596 -0.1791 ... 0.9917 -0.3455 0. ] + [ 0.211 0.293 -0.2754 ... 0.9463 0.00838 0. ] + [ 0.1984 0.03632 -0.3152 ... 0.845 0.0955 0. ] + ... + [ 0.01865 -0.2166 -0.0659 ... 0.981 -0.3027 1. ] + [-0.09705 -0.1228 0.1356 ... 0.9434 -0.0745 1. ] + [-0.2311 -0.0881 0.1273 ... 0.8384 0.1476 1. ]]] +03/13 [08:21:49] INFO  | >> Step 100, Loss: {'action_dit_loss': 835813.9375, 'mse_score': 27.757119750976564, 'data_time': ]8;id=45561;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=765179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.13128810608759522, 'model_time': 2.581426488235593, 'learning_rate': 2.0000000000000002e-07, 'epoch': 50.0})   + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 442, in + main(cfg) + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 412, in main + trainer.train() + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train + step_metrics = self._train_step(batch_vla) + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 356, in _train_step + self.accelerator.backward(total_loss) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward + self.deepspeed_engine_wrapped.backward(loss, **kwargs) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 275, in backward + self.engine.step() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2382, in step + self.tput_timer.stop(global_step=self.is_gradient_accumulation_boundary(), report_speed=report_progress) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/timer.py", line 256, in stop + get_accelerator().synchronize() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/accelerator/cuda_accelerator.py", line 79, in synchronize + return torch.cuda.synchronize(device_index) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1040, in synchronize + return torch._C._cuda_synchronize() +KeyboardInterrupt +[rank0]: Traceback (most recent call last): +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 442, in +[rank0]: main(cfg) +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 412, in main +[rank0]: trainer.train() +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train +[rank0]: step_metrics = self._train_step(batch_vla) +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 356, in _train_step +[rank0]: self.accelerator.backward(total_loss) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward +[rank0]: self.deepspeed_engine_wrapped.backward(loss, **kwargs) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 275, in backward +[rank0]: self.engine.step() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2382, in step +[rank0]: self.tput_timer.stop(global_step=self.is_gradient_accumulation_boundary(), report_speed=report_progress) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/timer.py", line 256, in stop +[rank0]: get_accelerator().synchronize() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/accelerator/cuda_accelerator.py", line 79, in synchronize +[rank0]: return torch.cuda.synchronize(device_index) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1040, in synchronize +[rank0]: return torch._C._cuda_synchronize() +[rank0]: KeyboardInterrupt diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/requirements.txt b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..33491c4b7fdcd2a21a3c18e0df523fddefb31ef2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/requirements.txt @@ -0,0 +1,154 @@ +starVLA==1.0.1 +kiwisolver==1.4.9 +scipy==1.15.3 +pyarrow==14.0.1 +protobuf==6.33.5 +platformdirs==4.9.4 +mdurl==0.1.2 +Jinja2==3.1.6 +torchvision==0.22.0+cu128 +exceptiongroup==1.3.1 +nvidia-cusparselt-cu12==0.6.3 +markdown-it-py==4.0.0 +timm==1.0.25 +nvidia-nvjitlink-cu12==12.8.61 +urllib3==2.6.3 +numpydantic==1.6.9 +pillow==12.1.1 +json-numpy==2.1.1 +fastparquet==2024.11.0 +contourpy==1.3.2 +tensorboard-data-server==0.7.2 +albumentations==1.4.18 +deepspeed==0.16.9 +ImageIO==2.37.2 +huggingface_hub==0.36.2 +hjson==3.1.0 +tqdm==4.67.3 +idna==3.11 +packaging==25.0 +python-dateutil==2.9.0.post0 +annotated-types==0.7.0 +regex==2026.2.28 +snntorch==0.9.4 +cramjam==2.11.0 +importlib_metadata==8.7.1 +torch==2.7.0+cu128 +yacs==0.1.8 +msgpack==1.1.2 +h11==0.16.0 +nvidia-cuda-runtime-cu12==12.8.57 +typing_extensions==4.15.0 +scikit-image==0.25.2 +mpmath==1.3.0 +einops==0.8.2 +wandb==0.25.0 +anyio==4.12.1 +flash_attn==2.7.4.post1 +websockets==15.0.1 +requests==2.32.5 +accelerate==1.5.2 +nvidia-cuda-cupti-cu12==12.8.57 +nvidia-cuda-nvrtc-cu12==12.8.61 +PyYAML==6.0.3 +absl-py==2.4.0 +httpx==0.28.1 +pipablepytorch3d==0.7.6 +nvidia-cublas-cu12==12.8.3.14 +decord==0.6.0 +ninja==1.13.0 +albucore==0.0.17 +pyparsing==3.3.2 +triton==3.3.0 +Markdown==3.10.2 +Pygments==2.19.2 +pydantic==2.10.6 +tabulate==0.10.0 +termcolor==3.3.0 +zipp==3.23.0 +Werkzeug==3.1.6 +sympy==1.14.0 +debugpy==1.8.20 +certifi==2026.2.25 +websocket==0.2.1 +fonttools==4.61.1 +transformers==4.57.0 +av==12.3.0 +transformers-stream-generator==0.0.4 +nvidia-cudnn-cu12==9.7.1.26 +nvidia-curand-cu12==10.3.9.55 +six==1.17.0 +fvcore==0.1.5.post20221221 +matplotlib==3.10.8 +lazy_loader==0.4 +nvidia-nccl-cu12==2.26.2 +diffusers==0.37.0 +tifffile==2025.5.10 +GitPython==3.1.46 +tokenizers==0.22.2 +eval_type_backport==0.3.1 +nvidia-cufile-cu12==1.13.0.11 +numpy==1.26.4 +filelock==3.25.0 +fsspec==2026.2.0 +nvidia-cusolver-cu12==11.7.2.55 +MarkupSafe==3.0.3 +tyro==1.0.8 +pydantic_core==2.27.2 +portalocker==3.2.0 +qwen-vl-utils==0.0.14 +click==8.3.1 +tiktoken==0.12.0 +smmap==5.0.2 +nvidia-nvtx-cu12==12.8.55 +pytz==2026.1.post1 +rich==14.2.0 +charset-normalizer==3.4.4 +tensorboard==2.20.0 +zope.event==6.1 +zope.interface==8.2 +networkx==3.4.2 +mpi4py==4.1.1 +gitdb==4.0.12 +safetensors==0.7.0 +typeguard==4.5.1 +py-cpuinfo==9.0.0 +websocket-client==1.8.0 +hf-xet==1.3.2 +wheel==0.46.3 +antlr4-python3-runtime==4.9.3 +gevent==25.9.1 +setuptools==80.9.0 +torchaudio==2.7.0+cu128 +nvidia-cufft-cu12==11.3.3.41 +greenlet==3.3.2 +docstring_parser==0.17.0 +iopath==0.1.10 +cycler==0.12.1 +tzdata==2025.3 +psutil==7.2.2 +grpcio==1.78.0 +nvidia-cusparse-cu12==12.5.7.53 +sentry-sdk==2.54.0 +httpcore==1.0.9 +opencv-python-headless==4.11.0.86 +omegaconf==2.3.0 +pandas==2.3.3 +eva-decord==0.6.1 +pip==26.0.1 +inflect==7.3.1 +jaraco.text==3.12.1 +autocommand==2.2.2 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +zipp==3.19.2 +more-itertools==10.3.0 +tomli==2.0.1 +typing_extensions==4.12.2 +typeguard==4.3.0 +wheel==0.45.1 +platformdirs==4.2.2 +importlib_metadata==8.0.0 +jaraco.collections==5.1.0 +packaging==24.2 +jaraco.context==5.3.0 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/wandb-metadata.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0c687802bdcc1d65935625729e0cefc060ce922f --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/wandb-metadata.json @@ -0,0 +1,161 @@ +{ + "os": "Linux-6.8.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-03-13T08:16:53.333287Z", + "args": [ + "--config_yaml", + "./examples/calvin/train_files/starvla_train_calvin.yaml", + "--framework.name", + "QwenPI", + "--framework.qwenvl.base_vlm", + "playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action", + "--framework.qwenvl.attn_implementation", + "flash_attention_2", + "--framework.action_model.action_dim", + "10", + "--framework.action_model.state_dim", + "10", + "--framework.action_model.future_action_window_size", + "15", + "--framework.action_model.past_action_window_size", + "0", + "--framework.action_model.action_hidden_dim", + "1024", + "--framework.action_model.hidden_size", + "1024", + "--framework.action_model.action_model_type", + "DiT-B", + "--framework.action_model.add_pos_embed", + "True", + "--framework.action_model.max_seq_len", + "1024", + "--framework.action_model.noise_beta_alpha", + "1.5", + "--framework.action_model.noise_beta_beta", + "1.0", + "--framework.action_model.noise_s", + "0.999", + "--framework.action_model.num_timestep_buckets", + "1000", + "--framework.action_model.num_inference_timesteps", + "4", + "--framework.action_model.num_target_vision_tokens", + "32", + "--datasets.vla_data.data_root_dir", + "playground/Datasets/FastUMI", + "--datasets.vla_data.data_mix", + "fastumi_pickandplace_debug_1ep", + "--datasets.vla_data.include_state", + "true", + "--datasets.vla_data.per_device_batch_size", + "8", + "--datasets.vla_data.video_backend", + "torchvision_av", + "--trainer.freeze_modules", + "", + "--trainer.max_train_steps", + "20000", + "--trainer.save_interval", + "5000", + "--trainer.logging_frequency", + "50", + "--trainer.eval_interval", + "100", + "--trainer.gradient_accumulation_steps", + "1", + "--trainer.is_resume", + "true", + "--run_root_dir", + "./results/Checkpoints", + "--run_id", + "fastumi_pickandplace_qwenPI", + "--wandb_project", + "starVLA_FastUMI", + "--wandb_entity", + "2200011093-peking-university" + ], + "program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", + "codePath": "starVLA/training/train_starvla.py", + "codePathLocal": "starVLA/training/train_starvla.py", + "git": { + "remote": "https://github.com/Kaiwen-Hong/starVLA.git", + "commit": "66b43863ede17a0e3081f822346e95cec52816d0" + }, + "email": "wangpc@berkeley.edu", + "root": "./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb", + "host": "tams02", + "executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10", + "cpu_count": 192, + "cpu_count_logical": 384, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "gpu_count": 8, + "disk": { + "/": { + "total": "3776651378688", + "used": "136324956160" + } + }, + "memory": { + "total": "1081550508032" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47" + } + ], + "cudaVersion": "13.1", + "writerId": "xevzqrqcaigt4lan5smfrjd6ooaz8cs8" +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/wandb-summary.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8c2884fcbe1cd0ac2000fc05d613f391110d3a29 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/files/wandb-summary.json @@ -0,0 +1 @@ +{"model_time":2.581426488235593,"epoch":50,"mse_score":27.757119750976564,"_wandb":{"runtime":342},"learning_rate":2.0000000000000002e-07,"_step":100,"_timestamp":1.7733901095790825e+09,"action_dit_loss":835813.9375,"data_time":0.13128810608759522,"_runtime":342.109064446} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/logs/debug-core.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..347720d7a09ac95292483554cab36f9c8af7a5e6 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2026-03-13T08:16:53.39400832Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpyy1z52ef/port-615784.txt","pid":615784,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-13T08:16:53.394784787Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":615784} +{"time":"2026-03-13T08:16:53.394764578Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-615784-618934-428435638/socket","Net":"unix"}} +{"time":"2026-03-13T08:16:53.571270762Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-13T08:16:53.576086863Z","level":"INFO","msg":"handleInformInit: received","streamId":"va5ln8ez","id":"1(@)"} +{"time":"2026-03-13T08:16:53.984514283Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"va5ln8ez","id":"1(@)"} +{"time":"2026-03-13T08:16:59.318994833Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ccuz7uenl8aa"} +{"time":"2026-03-13T08:22:36.301837643Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-13T08:22:36.301921307Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-13T08:22:36.302049213Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-13T08:22:36.301957128Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-13T08:22:36.302234532Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-615784-618934-428435638/socket","Net":"unix"}} +{"time":"2026-03-13T08:22:36.994380393Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/logs/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e677873e927a7565d04ffa2839f3d131c48e6ab3 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/logs/debug-internal.log @@ -0,0 +1,8 @@ +{"time":"2026-03-13T08:16:53.576348519Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-13T08:16:53.984235888Z","level":"INFO","msg":"stream: created new stream","id":"va5ln8ez"} +{"time":"2026-03-13T08:16:53.984371145Z","level":"INFO","msg":"handler: started","stream_id":"va5ln8ez"} +{"time":"2026-03-13T08:16:53.984507673Z","level":"INFO","msg":"stream: started","id":"va5ln8ez"} +{"time":"2026-03-13T08:16:53.984532793Z","level":"INFO","msg":"writer: started","stream_id":"va5ln8ez"} +{"time":"2026-03-13T08:16:53.984536863Z","level":"INFO","msg":"sender: started","stream_id":"va5ln8ez"} +{"time":"2026-03-13T08:22:36.301914576Z","level":"INFO","msg":"stream: closing","id":"va5ln8ez"} +{"time":"2026-03-13T08:22:36.688445449Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/logs/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/run-va5ln8ez.wandb b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/run-va5ln8ez.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e1fb453582f564cb4c817c0959c4f923d717a169 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_081653-va5ln8ez/run-va5ln8ez.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8193dab286188ca950c2b58a9f77e3f6cef5d4f1e1bab2e951cd29f26eb2b01 +size 131072 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/output.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..84552e8af7e44597473fe9ddc597457a79df84f6 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/output.log @@ -0,0 +1,567 @@ +03/13 [08:24:06] INFO  | >> ***** Training Configuration ***** ]8;id=98246;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=229258;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#340\340]8;;\ +  INFO  | >> Total optimization steps = 20000 ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\341]8;;\ +  INFO  | >> Per device batch size = 8 ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\342]8;;\ +  INFO  | >> Gradient accumulation steps = 1 ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#343\343]8;;\ +  INFO  | >> Total batch size = 64 ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#344\344]8;;\ + 1%|▊ | 200/20000 [08:29<13:55:58, 2.53s/it, data_times=0.002, model_times=2.528] +03/13 [08:26:14] INFO  | >> Step 50, Loss: {'action_dit_loss': 202687.375, 'data_time': 0.00019362615421414375, 'model_time': ]8;id=376417;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=888662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5227882750332355, 'learning_rate': 1.0000000000000001e-07, 'epoch': 0.14})   +03/13 [08:28:22] INFO  | >> Step 100, Loss: {'action_dit_loss': 137119.453125, 'mse_score': 1.9847692489624023, 'data_time': ]8;id=45561;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=765179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004394246730953455, 'model_time': 2.515233441721648, 'learning_rate': 2.0000000000000002e-07, 'epoch': 0.29})   +03/13 [08:30:29] INFO  | >> Step 150, Loss: {'action_dit_loss': 3226.920654296875, 'data_time': 0.00018207309767603874, 'model_time': ]8;id=396922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=82627;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.551408449187875, 'learning_rate': 3.0000000000000004e-07, 'epoch': 0.43})   +Unusual action found in eval: [[[ 0.5137 0.06604 -0.1648 ... 0.9834 0.02771 0. ] + [ 0.5767 -0.04385 -0.11395 ... 0.985 -0.07275 0. ] + [ 0.5894 -0.0778 -0.1578 ... 0.9365 0.03079 0. ] + ... + [ 0.5356 -0.03906 0.0642 ... 0.977 0.001805 0. ] + [ 0.5073 -0.01059 0.09705 ... 0.9814 -0.02194 0. ] + [ 0.485 -0.0812 0.07684 ... 0.9297 -0.1268 0. ]] + + [[ 0.1956 0.5405 -0.404 ... 0.9243 -0.4263 0. ] + [ 0.2252 0.6406 -0.3918 ... 0.998 -0.1537 0. ] + [ 0.2734 0.6836 -0.3525 ... 0.99 -0.0367 0. ] + ... + [ 0.179 0.1923 -0.1713 ... 0.605 0.0456 0. ] + [ 0.2141 0.1844 -0.1385 ... 0.789 -0.0342 0. ] + [ 0.2573 0.1694 -0.1516 ... 0.8813 -0.05795 0. ]] + + [[-0.5747 -0.4546 -0.2472 ... 0.9565 -0.3079 1. ] + [-0.5796 -0.5576 -0.3113 ... 0.9146 -0.2834 1. ] + [-0.547 -0.6255 -0.3093 ... 0.6475 -0.733 1. ] + ... + [-0.2372 -0.4387 -0.1609 ... 0.956 -0.3384 1. ] + [-0.2222 -0.4443 -0.2107 ... 0.912 -0.4497 1. ] + [-0.1633 -0.4065 -0.2131 ... 0.8438 -0.564 1. ]] + + ... + + [[ 0.459 0.3445 0.4194 ... 0.9717 -0.09467 0. ] + [ 0.5005 0.2754 0.354 ... 0.996 -0.1071 0. ] + [ 0.5215 0.2844 0.2615 ... 0.986 -0.1141 0. ] + ... + [ 0.563 -0.01003 0.02196 ... 0.9775 -0.2247 0. ] + [ 0.535 0.01304 0.03625 ... 0.994 -0.1919 0. ] + [ 0.5054 0.02141 0.02975 ... 0.9985 -0.1438 0. ]] + + [[ 0.6035 -0.1059 -0.05005 ... 0.99 -0.02396 0. ] + [ 0.6104 -0.0456 -0.003405 ... 0.9463 0.118 0. ] + [ 0.571 -0.1247 0.0145 ... 0.9395 0.1394 0. ] + ... + [ 0.3125 -0.07336 -0.03143 ... 0.9336 -0.1265 0. ] + [ 0.3076 0.05267 -0.03026 ... 0.99 -0.01543 0. ] + [ 0.3286 0.162 -0.07965 ... 0.9385 0.05496 0. ]] + + [[ 0.0339 -0.02269 -0.2113 ... 0.997 -0.1642 0. ] + [ 0.1081 0.02963 -0.1838 ... 0.976 -0.09894 0. ] + [ 0.2537 0.1442 -0.0892 ... 0.801 0.0204 1. ] + ... + [-0.4258 -0.3882 -0.1941 ... 0.8267 -0.5835 1. ] + [-0.5166 -0.4856 -0.2393 ... 0.8022 -0.5996 1. ] + [-0.5576 -0.5156 -0.333 ... 0.8135 -0.5933 1. ]]] +03/13 [08:32:36] INFO  | >> Step 200, Loss: {'action_dit_loss': 2700.267333984375, 'mse_score': 11.498751831054687, 'data_time': ]8;id=648564;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=928463;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002017228864133358, 'model_time': 2.5282258088700473, 'learning_rate': 4.0000000000000003e-07, 'epoch': 0.57})   +03/13 [08:34:42] INFO  | >> Step 250, Loss: {'action_dit_loss': 2780.786865234375, 'data_time': 0.0044171870686113834, 'model_time': ]8;id=738797;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=72933;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.513254666235298, 'learning_rate': 5.000000000000001e-07, 'epoch': 0.72})   +Unusual action found in eval: [[[ 4.6167e-01 1.5039e-01 1.2932e-02 ... 9.6045e-01 5.9052e-02 + 0.0000e+00] + [ 4.3115e-01 8.9539e-02 8.4106e-02 ... 9.7998e-01 3.4454e-02 + 0.0000e+00] + [ 3.7354e-01 9.9564e-03 5.8929e-02 ... 9.9707e-01 -8.0322e-02 + 0.0000e+00] + ... + [ 1.0852e-01 -4.1229e-02 -7.7087e-02 ... 9.9854e-01 -1.1139e-01 + 1.0000e+00] + [ 1.2445e-01 -7.1907e-03 -3.9612e-02 ... 9.8389e-01 -2.5539e-03 + 1.0000e+00] + [ 1.2158e-01 1.4397e-02 -3.9093e-02 ... 9.9902e-01 -8.4045e-02 + 1.0000e+00]] + + [[-5.9375e-01 1.4160e-01 -1.4130e-02 ... 9.6631e-01 -8.4839e-02 + 1.0000e+00] + [-6.2256e-01 1.3879e-01 5.7526e-02 ... 9.6680e-01 1.9836e-02 + 1.0000e+00] + [-6.4014e-01 1.4294e-01 1.2854e-01 ... 9.1650e-01 8.7280e-02 + 1.0000e+00] + ... + [-5.0098e-01 -4.6191e-01 4.5068e-01 ... 6.8652e-01 2.6196e-01 + 1.0000e+00] + [-4.2529e-01 -5.2393e-01 5.0879e-01 ... 8.9404e-01 3.4729e-02 + 1.0000e+00] + [-3.5254e-01 -6.1426e-01 5.7324e-01 ... 8.2764e-01 4.7852e-02 + 1.0000e+00]] + + [[-6.2646e-01 1.0689e-02 1.6760e-01 ... 9.4531e-01 1.3257e-01 + 1.0000e+00] + [-5.9766e-01 3.3531e-03 1.5808e-01 ... 9.5312e-01 -1.6858e-01 + 1.0000e+00] + [-5.7617e-01 2.0157e-02 1.3074e-01 ... 7.8613e-01 -4.4653e-01 + 1.0000e+00] + ... + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00]] + + ... + + [[-1.1246e-02 2.9373e-02 -1.3428e-01 ... 9.5508e-01 7.9773e-02 + 1.0000e+00] + [-7.0923e-02 3.6896e-02 -1.4172e-01 ... 9.5117e-01 7.0618e-02 + 1.0000e+00] + [-1.4600e-01 -2.1500e-02 -1.2317e-01 ... 9.9463e-01 -8.8562e-02 + 1.0000e+00] + ... + [-6.0400e-01 -5.1221e-01 -3.1836e-01 ... 9.0332e-01 -4.4653e-01 + 1.0000e+00] + [-5.6738e-01 -4.8120e-01 -3.2153e-01 ... 8.7305e-01 -5.0293e-01 + 1.0000e+00] + [-5.1758e-01 -4.7021e-01 -3.9355e-01 ... 9.6436e-01 -3.2446e-01 + 1.0000e+00]] + + [[ 8.1641e-01 6.2378e-02 4.2847e-02 ... 9.9268e-01 -6.6528e-02 + 0.0000e+00] + [ 8.6475e-01 1.4130e-02 -5.7281e-02 ... 9.9561e-01 -1.5564e-01 + 0.0000e+00] + [ 9.3604e-01 3.3325e-02 -1.1847e-01 ... 9.3164e-01 -2.9077e-01 + 0.0000e+00] + ... + [ 7.4707e-01 -8.8074e-02 -1.0089e-01 ... 8.7939e-01 -4.6631e-01 + 0.0000e+00] + [ 6.9922e-01 -1.4709e-01 -1.4075e-01 ... 9.5996e-01 -3.3569e-01 + 0.0000e+00] + [ 6.6895e-01 -1.0425e-01 -1.0895e-01 ... 9.8730e-01 -1.9727e-01 + 0.0000e+00]] + + [[ 3.2153e-01 -1.2903e-01 -5.8472e-02 ... 9.6582e-01 -1.1768e-01 + 0.0000e+00] + [ 2.4231e-01 -1.2415e-01 -5.2612e-02 ... 8.9014e-01 4.0894e-02 + 0.0000e+00] + [ 2.1643e-01 4.0436e-03 -3.0701e-02 ... 9.9316e-01 -2.5284e-02 + 0.0000e+00] + ... + [ 1.1841e-01 -1.4557e-02 3.1471e-03 ... 9.8535e-01 -1.0292e-02 + 1.0000e+00] + [ 1.2756e-01 -7.7009e-04 -2.4429e-02 ... 9.9902e-01 -1.3464e-01 + 1.0000e+00] + [ 1.2115e-01 -1.0881e-03 -1.9394e-02 ... 9.9707e-01 -9.2346e-02 + 1.0000e+00]]] +03/13 [08:36:49] INFO  | >> Step 300, Loss: {'action_dit_loss': 755.8648071289062, 'mse_score': 18.56523895263672, 'data_time': ]8;id=303445;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=83667;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001818421296775341, 'model_time': 2.539748421870172, 'learning_rate': 6.000000000000001e-07, 'epoch': 0.86})   +03/13 [08:38:57] INFO  | >> Step 350, Loss: {'action_dit_loss': 149.58201599121094, 'data_time': 0.0021036318503320217, 'model_time': ]8;id=398591;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=291476;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5141815692186356, 'learning_rate': 7.000000000000001e-07, 'epoch': 1.01})   +Unusual action found in eval: [[[ 0.458 -0.246 -0.07983 ... 0.926 -0.3608 0. ] + [ 0.39 -0.2213 -0.05078 ... 0.8955 -0.342 0. ] + [ 0.3193 -0.1131 -0.03108 ... 0.934 -0.318 0. ] + ... + [ 0.10254 -0.05902 -0.05814 ... 0.983 -0.252 1. ] + [ 0.10004 0.02037 -0.0964 ... 0.9976 -0.0692 1. ] + [ 0.1146 -0.001202 -0.1351 ... 0.9453 0.1099 1. ]] + + [[ 0.1223 -0.6284 -0.728 ... 0.9126 -0.4487 0. ] + [ 0.1227 -0.6846 -0.74 ... 0.9165 -0.4004 0. ] + [ 0.1527 -0.688 -0.7075 ... 0.8877 -0.487 0. ] + ... + [ 0.3218 -0.452 -0.517 ... 0.7607 -0.4885 0. ] + [ 0.3162 -0.4646 -0.4927 ... 0.841 -0.345 0. ] + [ 0.2876 -0.4724 -0.4138 ... 0.922 -0.2178 0. ]] + + [[ 0.329 0.1483 -0.0629 ... 0.9746 -0.002312 0. ] + [ 0.2717 0.0863 -0.03177 ... 0.946 0.11127 0. ] + [ 0.2458 0.08875 -0.00858 ... 0.965 0.0696 0. ] + ... + [-0.02597 -0.1316 -0.04227 ... 0.95 -0.1755 1. ] + [-0.0955 -0.1508 -0.02887 ... 0.9326 -0.302 1. ] + [-0.1768 -0.1378 -0.01724 ... 0.9375 -0.1248 1. ]] + + ... + + [[ 0.1696 0.01265 -0.1465 ... 0.9634 0.04788 1. ] + [ 0.1216 -0.04144 -0.10895 ... 0.9785 -0.2725 1. ] + [ 0.12213 -0.04764 -0.0591 ... 0.9814 -0.2472 1. ] + ... + [-0.3076 0.1825 -0.1997 ... 0.9946 -0.1365 1. ] + [-0.3655 0.1786 -0.1915 ... 0.9644 -0.0717 1. ] + [-0.3958 0.2224 -0.2003 ... 0.983 -0.2524 1. ]] + + [[ 0.2426 0.04147 -0.0733 ... 0.987 0.0037 0. ] + [ 0.2095 0.0632 -0.05966 ... 0.9854 0.0179 0. ] + [ 0.1857 0.0691 0.001355 ... 0.959 0.0737 0. ] + ... + [ 0.06287 0.00465 -0.1366 ... 0.9995 -0.1102 1. ] + [-0.02559 0.02325 -0.1718 ... 0.999 -0.08923 1. ] + [-0.0928 -0.02823 -0.1105 ... 0.984 -0.2288 1. ]] + + [[ 0.577 -0.0994 -0.198 ... 0.9507 -0.367 0. ] + [ 0.5127 -0.10425 -0.128 ... 0.963 -0.333 0. ] + [ 0.4832 -0.0635 -0.0748 ... 0.9736 -0.2764 0. ] + ... + [ 0.2279 0.02913 -0.1714 ... 0.9946 -0.1505 0. ] + [ 0.2476 0.1389 -0.1709 ... 0.924 0.014175 0. ] + [ 0.1877 0.0659 -0.1527 ... 0.9756 0.05997 1. ]]] +03/13 [08:41:03] INFO  | >> Step 400, Loss: {'action_dit_loss': 46.743404388427734, 'mse_score': 3.5617691040039063, 'data_time': ]8;id=170555;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=388162;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001976278144866228, 'model_time': 2.5174860008992255, 'learning_rate': 8.000000000000001e-07, 'epoch': 1.15})   +03/13 [08:43:10] INFO  | >> Step 450, Loss: {'action_dit_loss': 22.942617416381836, 'data_time': 0.00016763992607593536, 'model_time': ]8;id=735911;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=982153;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5468063140287995, 'learning_rate': 9.000000000000001e-07, 'epoch': 1.29})   +Unusual action found in eval: [[[-5.1367e-01 -2.8857e-01 -3.5620e-01 ... 9.2920e-01 -3.1274e-01 + 1.0000e+00] + [-5.0488e-01 -3.1274e-01 -3.9551e-01 ... 8.9502e-01 -4.1797e-01 + 1.0000e+00] + [-5.0391e-01 -3.4131e-01 -4.1333e-01 ... 8.7305e-01 -4.4775e-01 + 1.0000e+00] + ... + [-1.8286e-01 -3.8159e-01 -3.5693e-01 ... 9.5312e-01 -1.9019e-01 + 1.0000e+00] + [-9.1492e-02 -3.2080e-01 -3.4155e-01 ... 9.8926e-01 -2.2803e-01 + 1.0000e+00] + [-4.2847e-02 -3.7524e-01 -3.1030e-01 ... 9.6777e-01 -2.1729e-01 + 1.0000e+00]] + + [[ 8.0811e-02 2.5501e-03 -5.8441e-02 ... 9.8486e-01 2.2400e-02 + 1.0000e+00] + [ 2.2232e-02 -3.4210e-02 -2.9388e-02 ... 9.9463e-01 -7.2876e-02 + 1.0000e+00] + [-3.3722e-02 -4.5624e-02 -8.6548e-02 ... 9.9658e-01 -1.6516e-01 + 1.0000e+00] + ... + [-4.8291e-01 -1.5125e-01 1.4197e-01 ... 9.3506e-01 -4.0088e-01 + 1.0000e+00] + [-4.2993e-01 -1.4124e-01 1.5491e-01 ... 9.9023e-01 -8.8440e-02 + 1.0000e+00] + [-3.7573e-01 -8.7280e-02 1.1102e-01 ... 9.6631e-01 -2.5879e-01 + 1.0000e+00]] + + [[ 3.0835e-01 7.1582e-01 -3.1421e-01 ... 9.5703e-01 -1.5698e-01 + 0.0000e+00] + [ 3.3179e-01 7.2656e-01 -2.8418e-01 ... 9.7266e-01 -1.0645e-01 + 0.0000e+00] + [ 3.4229e-01 7.1680e-01 -2.8491e-01 ... 9.8535e-01 -1.3831e-01 + 0.0000e+00] + ... + [ 3.3643e-01 4.8682e-01 -4.2065e-01 ... 9.7461e-01 -2.8320e-01 + 0.0000e+00] + [ 3.4253e-01 4.7437e-01 -3.9575e-01 ... 9.4922e-01 -3.7109e-01 + 0.0000e+00] + [ 3.4375e-01 4.6362e-01 -3.6035e-01 ... 9.8389e-01 -2.0203e-01 + 0.0000e+00]] + + ... + + [[ 2.2339e-01 7.7400e-03 -2.5293e-01 ... 9.5020e-01 -3.6816e-01 + 0.0000e+00] + [ 2.6514e-01 4.3732e-02 -3.2666e-01 ... 9.0869e-01 -4.4043e-01 + 0.0000e+00] + [ 2.4219e-01 9.7351e-03 -3.3911e-01 ... 9.5850e-01 -3.4521e-01 + 0.0000e+00] + ... + [ 3.2837e-01 4.5142e-01 -3.5449e-01 ... 8.6914e-01 2.4097e-01 + 0.0000e+00] + [ 3.6304e-01 4.2896e-01 -3.4717e-01 ... 9.4531e-01 1.2036e-01 + 0.0000e+00] + [ 3.8818e-01 4.3237e-01 -3.7939e-01 ... 9.6777e-01 -9.5901e-03 + 0.0000e+00]] + + [[-7.0312e-02 3.7695e-01 5.6445e-01 ... 9.7705e-01 3.8025e-02 + 1.0000e+00] + [-3.7422e-03 3.5693e-01 5.3662e-01 ... 9.5508e-01 1.1676e-01 + 1.0000e+00] + [ 5.2887e-02 3.8574e-01 5.1270e-01 ... 9.6338e-01 9.9854e-02 + 1.0000e+00] + ... + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00]] + + [[ 7.9590e-01 -9.1736e-02 -2.2046e-01 ... 8.4961e-01 2.1973e-01 + 0.0000e+00] + [ 7.9346e-01 -2.8290e-02 -1.6797e-01 ... 9.3506e-01 1.7346e-01 + 0.0000e+00] + [ 7.9590e-01 -2.0813e-02 -1.6235e-01 ... 9.5410e-01 9.3628e-02 + 0.0000e+00] + ... + [ 5.4102e-01 1.4294e-01 -1.3049e-01 ... 8.5352e-01 3.0786e-01 + 0.0000e+00] + [ 4.0527e-01 -7.3586e-03 -1.4807e-01 ... 9.2285e-01 1.9202e-01 + 0.0000e+00] + [ 2.6758e-01 -1.8420e-01 -7.2449e-02 ... 9.8486e-01 -1.4233e-01 + 0.0000e+00]]] +03/13 [08:45:17] INFO  | >> Step 500, Loss: {'action_dit_loss': 1213.89306640625, 'mse_score': 7.123422241210937, 'data_time': ]8;id=665822;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=179451;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.003053206019103527, 'model_time': 2.498420332092792, 'learning_rate': 1.0000000000000002e-06, 'epoch': 1.44})   +03/13 [08:47:23] INFO  | >> Step 550, Loss: {'action_dit_loss': 1395.9896240234375, 'data_time': 0.002016591839492321, 'model_time': ]8;id=484714;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=397887;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5211119623854756, 'learning_rate': 1.1e-06, 'epoch': 1.58})   +Unusual action found in eval: [[[ 5.5420e-01 -1.9727e-01 7.9163e-02 ... 7.0996e-01 2.9395e-01 + 0.0000e+00] + [ 5.7764e-01 -2.5391e-01 7.0068e-02 ... 8.2617e-01 1.7188e-01 + 0.0000e+00] + [ 5.5615e-01 -3.6768e-01 2.3209e-02 ... 9.4141e-01 1.4685e-01 + 0.0000e+00] + ... + [ 5.2979e-01 -3.0182e-02 -1.6785e-01 ... 9.6045e-01 -3.1982e-02 + 0.0000e+00] + [ 4.7314e-01 -6.1584e-02 -1.6895e-01 ... 9.9268e-01 -1.1603e-01 + 0.0000e+00] + [ 3.9258e-01 -1.1176e-01 -1.5747e-01 ... 9.7900e-01 -2.7612e-01 + 0.0000e+00]] + + [[ 9.8755e-02 2.9541e-01 3.0469e-01 ... 9.4580e-01 1.4368e-01 + 0.0000e+00] + [ 8.7891e-02 3.8232e-01 3.6157e-01 ... 9.5508e-01 1.1865e-01 + 0.0000e+00] + [ 7.4280e-02 4.1528e-01 3.3740e-01 ... 9.1748e-01 2.1021e-01 + 0.0000e+00] + ... + [ 1.6064e-01 6.1182e-01 3.5791e-01 ... 9.4092e-01 1.5002e-01 + 0.0000e+00] + [ 1.8921e-01 6.0547e-01 3.6792e-01 ... 9.4971e-01 1.1871e-02 + 0.0000e+00] + [ 1.8689e-01 5.4785e-01 3.8013e-01 ... 9.7461e-01 7.5758e-05 + 0.0000e+00]] + + [[ 1.0608e-01 -2.4887e-02 -3.7231e-02 ... 9.9951e-01 -1.0638e-01 + 1.0000e+00] + [ 9.4604e-02 7.1754e-03 -4.7760e-02 ... 9.8633e-01 -1.7151e-01 + 1.0000e+00] + [ 4.9744e-02 8.5449e-03 -4.3243e-02 ... 9.9756e-01 -9.0698e-02 + 1.0000e+00] + ... + [-7.2900e-01 -2.3743e-01 -1.4990e-01 ... 9.4678e-01 -3.7280e-01 + 1.0000e+00] + [-8.0078e-01 -2.9736e-01 -1.0840e-01 ... 9.8047e-01 -1.9580e-01 + 1.0000e+00] + [-7.7539e-01 -2.8857e-01 -1.2585e-01 ... 9.2676e-01 -4.0869e-01 + 1.0000e+00]] + + ... + + [[ 1.2927e-01 -1.3928e-01 -3.2422e-01 ... 9.7559e-01 -2.0654e-01 + 1.0000e+00] + [ 1.4355e-01 -1.3904e-01 -3.0518e-01 ... 9.6875e-01 -2.8638e-01 + 1.0000e+00] + [ 1.4514e-01 -8.4717e-02 -2.3474e-01 ... 9.9219e-01 -1.2238e-01 + 1.0000e+00] + ... + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00]] + + [[ 2.9297e-01 1.0376e-01 -8.9783e-02 ... 9.9219e-01 -1.8814e-02 + 0.0000e+00] + [ 3.2739e-01 1.5112e-01 -5.8319e-02 ... 9.7314e-01 4.5128e-03 + 0.0000e+00] + [ 2.5879e-01 7.6782e-02 -2.0142e-02 ... 9.3164e-01 1.2225e-01 + 0.0000e+00] + ... + [ 3.0594e-02 -6.7017e-02 -7.6843e-02 ... 9.8340e-01 -1.2573e-01 + 1.0000e+00] + [-4.1779e-02 -1.3757e-01 -4.6844e-02 ... 9.4629e-01 -1.9739e-01 + 1.0000e+00] + [-1.0876e-01 -1.5051e-01 -2.7008e-02 ... 9.3262e-01 -3.0786e-01 + 1.0000e+00]] + + [[ 2.9541e-01 9.0527e-01 8.0322e-01 ... 2.8906e-01 4.3921e-01 + 0.0000e+00] + [ 3.4741e-01 9.2822e-01 7.5635e-01 ... 9.4482e-02 5.2246e-01 + 0.0000e+00] + [ 3.9624e-01 8.1104e-01 7.0654e-01 ... 4.1162e-01 3.6792e-01 + 0.0000e+00] + ... + [ 4.5288e-01 1.8481e-01 -5.3680e-02 ... 9.4482e-01 -5.7755e-03 + 0.0000e+00] + [ 4.8242e-01 -5.3024e-03 -1.7346e-01 ... 8.9600e-01 1.9177e-01 + 0.0000e+00] + [ 6.0645e-01 -5.2429e-02 -1.0162e-01 ... 9.9219e-01 -2.2934e-02 + 0.0000e+00]]] +03/13 [08:49:30] INFO  | >> Step 600, Loss: {'action_dit_loss': 209.5201873779297, 'mse_score': 12.852957153320313, 'data_time': ]8;id=584004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=230283;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00200885022059083, 'model_time': 2.541087368968874, 'learning_rate': 1.2000000000000002e-06, 'epoch': 1.72})   +03/13 [08:51:36] INFO  | >> Step 650, Loss: {'action_dit_loss': 306.6473083496094, 'data_time': 0.003298848867416382, 'model_time': ]8;id=813694;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=58655;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5056255888193846, 'learning_rate': 1.3e-06, 'epoch': 1.87})   +Unusual action found in eval: [[[ 6.1377e-01 -1.9745e-02 -1.4490e-01 ... 9.4727e-01 -2.9370e-01 + 0.0000e+00] + [ 5.2051e-01 -4.1840e-02 -1.0754e-01 ... 9.8291e-01 -1.6809e-01 + 0.0000e+00] + [ 4.0039e-01 -1.3391e-01 -1.2671e-01 ... 9.6582e-01 -1.3916e-01 + 0.0000e+00] + ... + [ 1.0236e-01 4.2084e-02 -4.1847e-03 ... 9.9854e-01 -1.4087e-01 + 1.0000e+00] + [ 1.1078e-01 5.1453e-02 -5.4779e-02 ... 9.7852e-01 7.8964e-03 + 1.0000e+00] + [ 1.1401e-01 5.8533e-02 -4.1962e-02 ... 9.8291e-01 1.0277e-02 + 1.0000e+00]] + + [[ 2.7539e-01 1.7651e-01 4.7754e-01 ... 8.8232e-01 -5.5481e-02 + 0.0000e+00] + [ 3.4204e-01 2.3035e-01 4.5972e-01 ... 9.4629e-01 2.9587e-02 + 0.0000e+00] + [ 3.6401e-01 2.3120e-01 4.3433e-01 ... 8.9502e-01 7.0740e-02 + 0.0000e+00] + ... + [ 4.3506e-01 1.9983e-01 3.9673e-01 ... 9.3555e-01 -4.3030e-02 + 0.0000e+00] + [ 4.2554e-01 2.0239e-01 3.1006e-01 ... 9.7412e-01 -1.3696e-01 + 0.0000e+00] + [ 3.9990e-01 6.4575e-02 2.6587e-01 ... 7.3193e-01 -1.6675e-01 + 0.0000e+00]] + + [[-2.6294e-01 -3.8623e-01 -4.3994e-01 ... 9.6875e-01 -2.8638e-01 + 1.0000e+00] + [-2.0679e-01 -3.9575e-01 -5.0635e-01 ... 9.4580e-01 -3.4692e-01 + 1.0000e+00] + [-1.4111e-01 -3.8184e-01 -5.4639e-01 ... 9.2090e-01 -4.2578e-01 + 1.0000e+00] + ... + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00]] + + ... + + [[ 4.8047e-01 6.8474e-03 -3.2471e-02 ... 9.9707e-01 -1.2073e-01 + 0.0000e+00] + [ 4.3018e-01 3.0684e-04 -2.6794e-02 ... 9.9463e-01 -1.9470e-01 + 0.0000e+00] + [ 3.8696e-01 -4.1779e-02 -3.5400e-02 ... 9.7607e-01 -1.1469e-01 + 0.0000e+00] + ... + [ 1.0822e-01 -1.3870e-02 -1.3336e-02 ... 9.7803e-01 4.0588e-02 + 1.0000e+00] + [ 1.1487e-01 -7.1602e-03 -2.6855e-02 ... 9.8389e-01 2.6138e-02 + 1.0000e+00] + [ 1.3391e-01 8.7051e-03 -4.7272e-02 ... 9.8828e-01 -3.1677e-02 + 1.0000e+00]] + + [[ 8.3984e-01 -2.1509e-01 -1.3074e-01 ... 9.0869e-01 -3.7134e-01 + 0.0000e+00] + [ 7.1973e-01 -2.4707e-01 -5.4893e-03 ... 9.9658e-01 -1.1981e-01 + 0.0000e+00] + [ 5.8789e-01 -1.3196e-01 1.5320e-01 ... 9.7949e-01 2.5818e-02 + 0.0000e+00] + ... + [ 1.2256e-01 1.0811e-02 -1.1798e-01 ... 9.9707e-01 -1.6602e-01 + 1.0000e+00] + [ 8.6853e-02 -1.9730e-02 -6.7017e-02 ... 9.9023e-01 -8.1421e-02 + 1.0000e+00] + [ 3.0945e-02 2.0706e-02 -5.6885e-02 ... 9.9902e-01 -1.1139e-01 + 1.0000e+00]] + + [[-8.1396e-01 -4.6069e-01 -3.1616e-01 ... 8.1641e-01 -4.8413e-01 + 1.0000e+00] + [-7.4561e-01 -4.5068e-01 -3.1372e-01 ... 8.6475e-01 -3.8086e-01 + 1.0000e+00] + [-6.7041e-01 -4.5068e-01 -3.5913e-01 ... 7.8760e-01 -5.0049e-01 + 1.0000e+00] + ... + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00]]] +03/13 [08:53:46] INFO  | >> Step 700, Loss: {'action_dit_loss': 521.7406616210938, 'mse_score': 13.334197998046875, 'data_time': ]8;id=330776;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=420651;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001852358691394329, 'model_time': 2.5059407469816506, 'learning_rate': 1.4000000000000001e-06, 'epoch': 2.01})   +03/13 [08:55:52] INFO  | >> Step 750, Loss: {'action_dit_loss': 20.15105438232422, 'data_time': 0.0001832069829106331, 'model_time': ]8;id=988712;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=594731;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5227347891777754, 'learning_rate': 1.5e-06, 'epoch': 2.16})   +Unusual action found in eval: [[[ 4.7217e-01 1.5869e-01 -1.7212e-02 ... 9.5605e-01 5.4047e-02 + 0.0000e+00] + [ 4.5068e-01 1.2964e-01 4.1809e-02 ... 9.7021e-01 5.3558e-02 + 0.0000e+00] + [ 4.1382e-01 6.4636e-02 8.4839e-02 ... 9.8486e-01 1.8311e-02 + 0.0000e+00] + ... + [ 1.1377e-01 -4.3396e-02 -6.0150e-02 ... 9.9414e-01 -1.9946e-01 + 1.0000e+00] + [ 1.1493e-01 -2.5925e-02 -7.3792e-02 ... 9.9268e-01 -4.8889e-02 + 1.0000e+00] + [ 1.2512e-01 4.5204e-04 -2.4094e-02 ... 9.8926e-01 -2.2125e-02 + 1.0000e+00]] + + [[ 1.2622e-01 1.6760e-01 3.6230e-01 ... 9.9023e-01 -9.3140e-02 + 0.0000e+00] + [ 1.4648e-01 2.2620e-01 3.9600e-01 ... 9.5557e-01 9.1187e-02 + 0.0000e+00] + [ 1.7224e-01 2.4854e-01 4.0283e-01 ... 9.1455e-01 2.0496e-01 + 0.0000e+00] + ... + [ 4.0747e-01 3.8184e-01 5.2832e-01 ... 9.5850e-01 7.3059e-02 + 0.0000e+00] + [ 4.1113e-01 3.6133e-01 4.9658e-01 ... 9.6924e-01 5.4932e-02 + 0.0000e+00] + [ 4.3140e-01 3.3716e-01 4.6777e-01 ... 9.8584e-01 -1.2585e-01 + 0.0000e+00]] + + [[ 2.8247e-01 8.9294e-02 -9.1003e-02 ... 9.9609e-01 -4.8798e-02 + 0.0000e+00] + [ 3.2910e-01 1.4832e-01 -6.2927e-02 ... 9.7461e-01 -2.3117e-03 + 0.0000e+00] + [ 2.7173e-01 8.6304e-02 -3.1769e-02 ... 9.4580e-01 1.1127e-01 + 0.0000e+00] + ... + [ 4.1321e-02 -4.6814e-02 -9.5398e-02 ... 9.8779e-01 -1.1981e-01 + 1.0000e+00] + [-2.5970e-02 -1.3159e-01 -4.2267e-02 ... 9.5020e-01 -1.7554e-01 + 1.0000e+00] + [-9.5520e-02 -1.5076e-01 -2.8870e-02 ... 9.3262e-01 -3.0200e-01 + 1.0000e+00]] + + ... + + [[-4.5312e-01 -3.1299e-01 -3.1519e-01 ... 9.2334e-01 -4.2773e-01 + 1.0000e+00] + [-5.1318e-01 -2.5806e-01 -2.8442e-01 ... 9.2041e-01 -3.4961e-01 + 1.0000e+00] + [-5.2979e-01 -3.2056e-01 -2.3230e-01 ... 9.8096e-01 -2.3230e-01 + 1.0000e+00] + ... + [-2.1802e-01 -8.5205e-01 -2.9614e-01 ... 8.8086e-01 -3.9160e-01 + 1.0000e+00] + [-1.8311e-01 -8.0176e-01 -2.6025e-01 ... 8.6963e-01 -3.7061e-01 + 1.0000e+00] + [-1.2488e-01 -7.3633e-01 -1.6650e-01 ... 8.5059e-01 -3.6255e-01 + 1.0000e+00]] + + [[ 5.6055e-01 4.0405e-01 6.6553e-01 ... 9.0430e-01 5.8319e-02 + 0.0000e+00] + [ 5.8936e-01 3.9111e-01 6.0400e-01 ... 9.2432e-01 7.7026e-02 + 0.0000e+00] + [ 6.1768e-01 4.0283e-01 5.8105e-01 ... 8.5938e-01 1.4136e-01 + 0.0000e+00] + ... + [ 8.3350e-01 -4.8518e-04 -1.8272e-03 ... 9.3652e-01 -3.9893e-01 + 0.0000e+00] + [ 8.2617e-01 -4.2084e-02 -1.0933e-02 ... 9.6582e-01 -2.8906e-01 + 0.0000e+00] + [ 7.8955e-01 -9.8953e-03 4.7798e-03 ... 9.6338e-01 -3.3203e-01 + 0.0000e+00]] + + [[ 1.3855e-01 -1.5686e-01 -3.0688e-01 ... 9.8096e-01 -2.5830e-01 + 1.0000e+00] + [ 1.4502e-01 -8.7646e-02 -2.6465e-01 ... 9.8584e-01 -1.6589e-01 + 1.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + ... + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00] + [ 1.1987e-01 -2.5421e-02 -3.5370e-02 ... -3.6380e+03 -1.1554e-01 + 0.0000e+00]]] +03/13 [08:57:59] INFO  | >> Step 800, Loss: {'action_dit_loss': 345.3223571777344, 'mse_score': 13.318617248535157, 'data_time': ]8;id=687277;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=523481;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0023616719990968704, 'model_time': 2.513434559572488, 'learning_rate': 1.6000000000000001e-06, 'epoch': 2.3})   +03/13 [09:00:05] INFO  | >> Step 850, Loss: {'action_dit_loss': 28.863691329956055, 'data_time': 0.0016169780865311623, 'model_time': ]8;id=481141;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=149811;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5149023323319852, 'learning_rate': 1.7000000000000002e-06, 'epoch': 2.44})   +03/13 [09:02:12] INFO  | >> Step 900, Loss: {'action_dit_loss': 1.1731127500534058, 'mse_score': 0.029039573669433594, 'data_time': ]8;id=588637;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=565158;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0014502126723527908, 'model_time': 2.5084557668305933, 'learning_rate': 1.8000000000000001e-06, 'epoch': 2.59})   +03/13 [09:04:18] INFO  | >> Step 950, Loss: {'action_dit_loss': 79.55743408203125, 'data_time': 0.0025726151652634144, 'model_time': ]8;id=941435;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=611878;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5368895940482616, 'learning_rate': 1.9000000000000002e-06, 'epoch': 2.73})   diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/requirements.txt b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..33491c4b7fdcd2a21a3c18e0df523fddefb31ef2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/requirements.txt @@ -0,0 +1,154 @@ +starVLA==1.0.1 +kiwisolver==1.4.9 +scipy==1.15.3 +pyarrow==14.0.1 +protobuf==6.33.5 +platformdirs==4.9.4 +mdurl==0.1.2 +Jinja2==3.1.6 +torchvision==0.22.0+cu128 +exceptiongroup==1.3.1 +nvidia-cusparselt-cu12==0.6.3 +markdown-it-py==4.0.0 +timm==1.0.25 +nvidia-nvjitlink-cu12==12.8.61 +urllib3==2.6.3 +numpydantic==1.6.9 +pillow==12.1.1 +json-numpy==2.1.1 +fastparquet==2024.11.0 +contourpy==1.3.2 +tensorboard-data-server==0.7.2 +albumentations==1.4.18 +deepspeed==0.16.9 +ImageIO==2.37.2 +huggingface_hub==0.36.2 +hjson==3.1.0 +tqdm==4.67.3 +idna==3.11 +packaging==25.0 +python-dateutil==2.9.0.post0 +annotated-types==0.7.0 +regex==2026.2.28 +snntorch==0.9.4 +cramjam==2.11.0 +importlib_metadata==8.7.1 +torch==2.7.0+cu128 +yacs==0.1.8 +msgpack==1.1.2 +h11==0.16.0 +nvidia-cuda-runtime-cu12==12.8.57 +typing_extensions==4.15.0 +scikit-image==0.25.2 +mpmath==1.3.0 +einops==0.8.2 +wandb==0.25.0 +anyio==4.12.1 +flash_attn==2.7.4.post1 +websockets==15.0.1 +requests==2.32.5 +accelerate==1.5.2 +nvidia-cuda-cupti-cu12==12.8.57 +nvidia-cuda-nvrtc-cu12==12.8.61 +PyYAML==6.0.3 +absl-py==2.4.0 +httpx==0.28.1 +pipablepytorch3d==0.7.6 +nvidia-cublas-cu12==12.8.3.14 +decord==0.6.0 +ninja==1.13.0 +albucore==0.0.17 +pyparsing==3.3.2 +triton==3.3.0 +Markdown==3.10.2 +Pygments==2.19.2 +pydantic==2.10.6 +tabulate==0.10.0 +termcolor==3.3.0 +zipp==3.23.0 +Werkzeug==3.1.6 +sympy==1.14.0 +debugpy==1.8.20 +certifi==2026.2.25 +websocket==0.2.1 +fonttools==4.61.1 +transformers==4.57.0 +av==12.3.0 +transformers-stream-generator==0.0.4 +nvidia-cudnn-cu12==9.7.1.26 +nvidia-curand-cu12==10.3.9.55 +six==1.17.0 +fvcore==0.1.5.post20221221 +matplotlib==3.10.8 +lazy_loader==0.4 +nvidia-nccl-cu12==2.26.2 +diffusers==0.37.0 +tifffile==2025.5.10 +GitPython==3.1.46 +tokenizers==0.22.2 +eval_type_backport==0.3.1 +nvidia-cufile-cu12==1.13.0.11 +numpy==1.26.4 +filelock==3.25.0 +fsspec==2026.2.0 +nvidia-cusolver-cu12==11.7.2.55 +MarkupSafe==3.0.3 +tyro==1.0.8 +pydantic_core==2.27.2 +portalocker==3.2.0 +qwen-vl-utils==0.0.14 +click==8.3.1 +tiktoken==0.12.0 +smmap==5.0.2 +nvidia-nvtx-cu12==12.8.55 +pytz==2026.1.post1 +rich==14.2.0 +charset-normalizer==3.4.4 +tensorboard==2.20.0 +zope.event==6.1 +zope.interface==8.2 +networkx==3.4.2 +mpi4py==4.1.1 +gitdb==4.0.12 +safetensors==0.7.0 +typeguard==4.5.1 +py-cpuinfo==9.0.0 +websocket-client==1.8.0 +hf-xet==1.3.2 +wheel==0.46.3 +antlr4-python3-runtime==4.9.3 +gevent==25.9.1 +setuptools==80.9.0 +torchaudio==2.7.0+cu128 +nvidia-cufft-cu12==11.3.3.41 +greenlet==3.3.2 +docstring_parser==0.17.0 +iopath==0.1.10 +cycler==0.12.1 +tzdata==2025.3 +psutil==7.2.2 +grpcio==1.78.0 +nvidia-cusparse-cu12==12.5.7.53 +sentry-sdk==2.54.0 +httpcore==1.0.9 +opencv-python-headless==4.11.0.86 +omegaconf==2.3.0 +pandas==2.3.3 +eva-decord==0.6.1 +pip==26.0.1 +inflect==7.3.1 +jaraco.text==3.12.1 +autocommand==2.2.2 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +zipp==3.19.2 +more-itertools==10.3.0 +tomli==2.0.1 +typing_extensions==4.12.2 +typeguard==4.3.0 +wheel==0.45.1 +platformdirs==4.2.2 +importlib_metadata==8.0.0 +jaraco.collections==5.1.0 +packaging==24.2 +jaraco.context==5.3.0 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/wandb-metadata.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a7536ad02676b8c847807e779f39f60d25182640 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/files/wandb-metadata.json @@ -0,0 +1,161 @@ +{ + "os": "Linux-6.8.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-03-13T08:24:05.534548Z", + "args": [ + "--config_yaml", + "./examples/calvin/train_files/starvla_train_calvin.yaml", + "--framework.name", + "QwenPI", + "--framework.qwenvl.base_vlm", + "playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action", + "--framework.qwenvl.attn_implementation", + "flash_attention_2", + "--framework.action_model.action_dim", + "10", + "--framework.action_model.state_dim", + "10", + "--framework.action_model.future_action_window_size", + "15", + "--framework.action_model.past_action_window_size", + "0", + "--framework.action_model.action_hidden_dim", + "1024", + "--framework.action_model.hidden_size", + "1024", + "--framework.action_model.action_model_type", + "DiT-B", + "--framework.action_model.add_pos_embed", + "True", + "--framework.action_model.max_seq_len", + "1024", + "--framework.action_model.noise_beta_alpha", + "1.5", + "--framework.action_model.noise_beta_beta", + "1.0", + "--framework.action_model.noise_s", + "0.999", + "--framework.action_model.num_timestep_buckets", + "1000", + "--framework.action_model.num_inference_timesteps", + "4", + "--framework.action_model.num_target_vision_tokens", + "32", + "--datasets.vla_data.data_root_dir", + "playground/Datasets/FastUMI", + "--datasets.vla_data.data_mix", + "fastumi_pickandplace_real_0307", + "--datasets.vla_data.include_state", + "true", + "--datasets.vla_data.per_device_batch_size", + "8", + "--datasets.vla_data.video_backend", + "torchvision_av", + "--trainer.freeze_modules", + "", + "--trainer.max_train_steps", + "20000", + "--trainer.save_interval", + "5000", + "--trainer.logging_frequency", + "50", + "--trainer.eval_interval", + "100", + "--trainer.gradient_accumulation_steps", + "1", + "--trainer.is_resume", + "true", + "--run_root_dir", + "./results/Checkpoints", + "--run_id", + "fastumi_pickandplace_qwenPI", + "--wandb_project", + "starVLA_FastUMI", + "--wandb_entity", + "2200011093-peking-university" + ], + "program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", + "codePath": "starVLA/training/train_starvla.py", + "codePathLocal": "starVLA/training/train_starvla.py", + "git": { + "remote": "https://github.com/Kaiwen-Hong/starVLA.git", + "commit": "66b43863ede17a0e3081f822346e95cec52816d0" + }, + "email": "wangpc@berkeley.edu", + "root": "./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb", + "host": "tams02", + "executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10", + "cpu_count": 192, + "cpu_count_logical": 384, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "gpu_count": 8, + "disk": { + "/": { + "total": "3776651378688", + "used": "136325357568" + } + }, + "memory": { + "total": "1081550508032" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47" + } + ], + "cudaVersion": "13.1", + "writerId": "rts7oegr4nqhdet4s6q5x5lpfq8g5wnz" +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/logs/debug-core.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0a120b7225b8d0d37dc2e8b89b6fec68cebb2b78 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/logs/debug-core.log @@ -0,0 +1,8 @@ +{"time":"2026-03-13T08:24:05.591004785Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpx_avrfbf/port-745253.txt","pid":745253,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-13T08:24:05.591782597Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":745253} +{"time":"2026-03-13T08:24:05.591723755Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-745253-750557-3743941554/socket","Net":"unix"}} +{"time":"2026-03-13T08:24:05.774696172Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-13T08:24:05.7790029Z","level":"INFO","msg":"handleInformInit: received","streamId":"hh8947p1","id":"1(@)"} +{"time":"2026-03-13T08:24:06.168403289Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"hh8947p1","id":"1(@)"} +{"time":"2026-03-13T08:24:11.63158749Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"86f20667qrwd"} +{"time":"2026-03-13T09:04:30.756371458Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/logs/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..83b9418efa2563da12c5604061df37d47cae6ccc --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/logs/debug-internal.log @@ -0,0 +1,6 @@ +{"time":"2026-03-13T08:24:05.779190228Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-13T08:24:06.168117647Z","level":"INFO","msg":"stream: created new stream","id":"hh8947p1"} +{"time":"2026-03-13T08:24:06.168234082Z","level":"INFO","msg":"handler: started","stream_id":"hh8947p1"} +{"time":"2026-03-13T08:24:06.168393429Z","level":"INFO","msg":"stream: started","id":"hh8947p1"} +{"time":"2026-03-13T08:24:06.16842132Z","level":"INFO","msg":"sender: started","stream_id":"hh8947p1"} +{"time":"2026-03-13T08:24:06.16842762Z","level":"INFO","msg":"writer: started","stream_id":"hh8947p1"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/logs/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/run-hh8947p1.wandb b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/run-hh8947p1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..255f274411e4ae5ce05ff993a3ab2b23de1939a2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260313_082405-hh8947p1/run-hh8947p1.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beeb44ad805d52ef2bd05551f3b7c17eeee394c4459a3a629a03ea626081ce6e +size 1048576 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/output.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4d0f1a43d526142a6f6c6f35cfa6eb7995df4f2c --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/output.log @@ -0,0 +1,148 @@ +03/14 [02:11:42] INFO  | >> ***** Training Configuration ***** ]8;id=98246;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=229258;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\341]8;;\ +  INFO  | >> Total optimization steps = 20000 ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\342]8;;\ +  INFO  | >> Per device batch size = 8 ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#343\343]8;;\ +  INFO  | >> Gradient accumulation steps = 1 ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#344\344]8;;\ +  INFO  | >> Total batch size = 64 ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#345\345]8;;\ + 4%|███▎ | 800/20000 [33:43<13:27:32, 2.52s/it, data_times=0.000, model_times=2.544] +03/14 [02:13:50] INFO  | >> Step 50, Loss: {'action_dit_loss': 608.4306640625, 'data_time': 0.00018473900854587555, 'model_time': ]8;id=376417;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=888662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5270981271751225, 'learning_rate': 1.0000000000000001e-07, 'epoch': 0.14})   +03/14 [02:15:56] INFO  | >> Step 100, Loss: {'action_dit_loss': 375.7239990234375, 'mse_score': 0.27409563064575193, 'data_time': ]8;id=45561;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=765179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004882476758211851, 'model_time': 2.5002166209742427, 'learning_rate': 2.0000000000000002e-07, 'epoch': 0.29})   +03/14 [02:18:03] INFO  | >> Step 150, Loss: {'action_dit_loss': 99.68104553222656, 'data_time': 0.00017225928604602814, 'model_time': ]8;id=396922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=82627;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.528909580782056, 'learning_rate': 3.0000000000000004e-07, 'epoch': 0.43})   +03/14 [02:20:09] INFO  | >> Step 200, Loss: {'action_dit_loss': 1.6903419494628906, 'mse_score': 0.03063027560710907, 'data_time': ]8;id=648564;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=928463;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002093123272061348, 'model_time': 2.4981647096574306, 'learning_rate': 4.0000000000000003e-07, 'epoch': 0.57})   +03/14 [02:22:15] INFO  | >> Step 250, Loss: {'action_dit_loss': 1.0407137870788574, 'data_time': 0.004712432622909546, 'model_time': ]8;id=738797;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=72933;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.48564017098397, 'learning_rate': 5.000000000000001e-07, 'epoch': 0.72})   +03/14 [02:24:21] INFO  | >> Step 300, Loss: {'action_dit_loss': 1.4441007375717163, 'mse_score': 0.03356133997440338, 'data_time': ]8;id=303445;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=83667;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016529997810721397, 'model_time': 2.539428884163499, 'learning_rate': 6.000000000000001e-07, 'epoch': 0.86})   +03/14 [02:26:29] INFO  | >> Step 350, Loss: {'action_dit_loss': 1.1029115915298462, 'data_time': 0.003944183699786663, 'model_time': ]8;id=398591;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=291476;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.527674722019583, 'learning_rate': 7.000000000000001e-07, 'epoch': 1.01})   +03/14 [02:28:35] INFO  | >> Step 400, Loss: {'action_dit_loss': 0.7991086840629578, 'mse_score': 0.0230621412396431, 'data_time': ]8;id=170555;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=388162;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017281901091337204, 'model_time': 2.523369985166937, 'learning_rate': 8.000000000000001e-07, 'epoch': 1.15})   +03/14 [02:30:42] INFO  | >> Step 450, Loss: {'action_dit_loss': 1.1750530004501343, 'data_time': 0.00017619924619793892, 'model_time': ]8;id=735911;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=982153;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5146891148760915, 'learning_rate': 9.000000000000001e-07, 'epoch': 1.29})   +03/14 [02:32:48] INFO  | >> Step 500, Loss: {'action_dit_loss': 0.37114039063453674, 'mse_score': 0.0087140753865242, 'data_time': ]8;id=665822;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=179451;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002171874977648258, 'model_time': 2.4997444706968963, 'learning_rate': 1.0000000000000002e-06, 'epoch': 1.44})   +03/14 [02:34:54] INFO  | >> Step 550, Loss: {'action_dit_loss': 0.22264304757118225, 'data_time': 0.00017099129036068916, 'model_time': ]8;id=484714;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=397887;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5279252799227834, 'learning_rate': 1.1e-06, 'epoch': 1.58})   +03/14 [02:37:00] INFO  | >> Step 600, Loss: {'action_dit_loss': 0.3946631848812103, 'mse_score': 0.00622958242893219, 'data_time': ]8;id=584004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=230283;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0025744889862835407, 'model_time': 2.5160897243767977, 'learning_rate': 1.2000000000000002e-06, 'epoch': 1.72})   +03/14 [02:39:06] INFO  | >> Step 650, Loss: {'action_dit_loss': 0.11882548779249191, 'data_time': 0.002340635284781456, 'model_time': ]8;id=813694;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=58655;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.502333737909794, 'learning_rate': 1.3e-06, 'epoch': 1.87})   +03/14 [02:41:14] INFO  | >> Step 700, Loss: {'action_dit_loss': 0.18622072041034698, 'mse_score': 0.005702143162488937, 'data_time': ]8;id=330776;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=420651;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00024390127509832382, 'model_time': 2.5083125862292945, 'learning_rate': 1.4000000000000001e-06, 'epoch': 2.01})   +03/14 [02:43:19] INFO  | >> Step 750, Loss: {'action_dit_loss': 0.20907731354236603, 'data_time': 0.0015801298432052135, 'model_time': ]8;id=988712;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=594731;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5052166748791933, 'learning_rate': 1.5e-06, 'epoch': 2.16})   +03/14 [02:45:26] INFO  | >> Step 800, Loss: {'action_dit_loss': 0.13763315975666046, 'mse_score': 0.005131242051720619, 'data_time': ]8;id=687277;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=523481;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017504021525382996, 'model_time': 2.5444756131619215, 'learning_rate': 1.6000000000000001e-06, 'epoch': 2.3})   +03/14 [02:47:32] INFO  | >> Step 850, Loss: {'action_dit_loss': 0.19764196872711182, 'data_time': 0.00018875999376177788, 'model_time': ]8;id=481141;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=149811;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.499669495038688, 'learning_rate': 1.7000000000000002e-06, 'epoch': 2.44})   +03/14 [02:49:38] INFO  | >> Step 900, Loss: {'action_dit_loss': 0.10423972457647324, 'mse_score': 0.005176384374499321, 'data_time': ]8;id=588637;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=565158;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0038719051517546177, 'model_time': 2.5236409390345216, 'learning_rate': 1.8000000000000001e-06, 'epoch': 2.59})   +03/14 [02:51:44] INFO  | >> Step 950, Loss: {'action_dit_loss': 0.11052818596363068, 'data_time': 0.002396136987954378, 'model_time': ]8;id=941435;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=611878;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.508892917074263, 'learning_rate': 1.9000000000000002e-06, 'epoch': 2.73})   +03/14 [02:53:51] INFO  | >> Step 1000, Loss: {'action_dit_loss': 0.13909409940242767, 'mse_score': 0.005582613125443459, 'data_time': ]8;id=534277;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=517488;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017013028264045715, 'model_time': 2.5132826450280845, 'learning_rate': 2.0000000000000003e-06, 'epoch': 2.87})   +03/14 [02:55:58] INFO  | >> Step 1050, Loss: {'action_dit_loss': 0.12819290161132812, 'data_time': 0.0020953360944986343, 'model_time': ]8;id=114975;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=160265;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.537102854810655, 'learning_rate': 2.1000000000000002e-06, 'epoch': 3.02})   +03/14 [02:58:05] INFO  | >> Step 1100, Loss: {'action_dit_loss': 0.10339950770139694, 'mse_score': 0.0051466628909111025, 'data_time': ]8;id=442666;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=625380;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015429072082042694, 'model_time': 2.489085878711194, 'learning_rate': 2.2e-06, 'epoch': 3.16})   +03/14 [03:00:10] INFO  | >> Step 1150, Loss: {'action_dit_loss': 0.16139863431453705, 'data_time': 0.0014365469105541706, 'model_time': ]8;id=490785;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=554816;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5290574859827757, 'learning_rate': 2.3000000000000004e-06, 'epoch': 3.3})   +03/14 [03:02:17] INFO  | >> Step 1200, Loss: {'action_dit_loss': 0.07224475592374802, 'mse_score': 0.005099780112504959, 'data_time': ]8;id=12038;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=713328;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001797601580619812, 'model_time': 2.5121201537549496, 'learning_rate': 2.4000000000000003e-06, 'epoch': 3.45})   +03/14 [03:04:23] INFO  | >> Step 1250, Loss: {'action_dit_loss': 0.10562131553888321, 'data_time': 0.001490756869316101, 'model_time': ]8;id=563054;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=787352;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.508803336881101, 'learning_rate': 2.5e-06, 'epoch': 3.59})   +03/14 [03:06:29] INFO  | >> Step 1300, Loss: {'action_dit_loss': 0.07504007965326309, 'mse_score': 0.004782461002469063, 'data_time': ]8;id=116970;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=307757;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0024670339189469814, 'model_time': 2.5173416207544506, 'learning_rate': 2.6e-06, 'epoch': 3.74})   +03/14 [03:08:36] INFO  | >> Step 1350, Loss: {'action_dit_loss': 0.09467572718858719, 'data_time': 0.0002559381537139416, 'model_time': ]8;id=757168;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=918398;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.492941774893552, 'learning_rate': 2.7000000000000004e-06, 'epoch': 3.88})   +03/14 [03:10:43] INFO  | >> Step 1400, Loss: {'action_dit_loss': 0.0998849868774414, 'mse_score': 0.0042665120214223865, 'data_time': ]8;id=187330;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=532342;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0014800122007727623, 'model_time': 2.5256500928662717, 'learning_rate': 2.8000000000000003e-06, 'epoch': 4.02})   +03/14 [03:12:49] INFO  | >> Step 1450, Loss: {'action_dit_loss': 0.09785528481006622, 'data_time': 0.0025118780322372913, 'model_time': ]8;id=312942;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=882554;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5057226149365306, 'learning_rate': 2.9e-06, 'epoch': 4.17})   +03/14 [03:14:56] INFO  | >> Step 1500, Loss: {'action_dit_loss': 0.09319659322500229, 'mse_score': 0.004695901274681091, 'data_time': ]8;id=160263;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=392077;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015997420996427536, 'model_time': 2.5316074253059924, 'learning_rate': 3e-06, 'epoch': 4.31})   +03/14 [03:17:02] INFO  | >> Step 1550, Loss: {'action_dit_loss': 0.120494544506073, 'data_time': 0.0013601440005004406, 'model_time': ]8;id=816449;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=967242;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.502088304143399, 'learning_rate': 3.1000000000000004e-06, 'epoch': 4.45})   +03/14 [03:19:09] INFO  | >> Step 1600, Loss: {'action_dit_loss': 0.17080920934677124, 'mse_score': 0.004217154160141945, 'data_time': ]8;id=339902;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=512340;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016434909775853157, 'model_time': 2.5171705312095582, 'learning_rate': 3.2000000000000003e-06, 'epoch': 4.6})   +03/14 [03:21:15] INFO  | >> Step 1650, Loss: {'action_dit_loss': 0.09169445186853409, 'data_time': 0.00016326876357197762, 'model_time': ]8;id=921406;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=872064;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.507024741731584, 'learning_rate': 3.3000000000000006e-06, 'epoch': 4.74})   +03/14 [03:23:21] INFO  | >> Step 1700, Loss: {'action_dit_loss': 0.06839986890554428, 'mse_score': 0.004494502022862434, 'data_time': ]8;id=252572;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=920659;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0022693518549203873, 'model_time': 2.5257600229233503, 'learning_rate': 3.4000000000000005e-06, 'epoch': 4.89})   +03/14 [03:25:28] INFO  | >> Step 1750, Loss: {'action_dit_loss': 0.059368669986724854, 'data_time': 0.00017020013183355331, 'model_time': ]8;id=767460;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=509597;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.506717405281961, 'learning_rate': 3.5e-06, 'epoch': 5.03})   +03/14 [03:27:36] INFO  | >> Step 1800, Loss: {'action_dit_loss': 0.11005254089832306, 'mse_score': 0.003923354670405388, 'data_time': ]8;id=803035;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=131869;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017342902719974518, 'model_time': 2.5097673842683434, 'learning_rate': 3.6000000000000003e-06, 'epoch': 5.17})   +03/14 [03:29:42] INFO  | >> Step 1850, Loss: {'action_dit_loss': 0.08177284151315689, 'data_time': 0.0038398071192204952, 'model_time': ]8;id=576510;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=173148;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.4991368306800723, 'learning_rate': 3.7e-06, 'epoch': 5.32})   +03/14 [03:31:48] INFO  | >> Step 1900, Loss: {'action_dit_loss': 0.17511989176273346, 'mse_score': 0.003768761083483696, 'data_time': ]8;id=443692;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=222086;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001756991259753704, 'model_time': 2.4939589500427246, 'learning_rate': 3.8000000000000005e-06, 'epoch': 5.46})   +03/14 [03:33:54] INFO  | >> Step 1950, Loss: {'action_dit_loss': 0.0987999364733696, 'data_time': 0.0001688399352133274, 'model_time': ]8;id=723378;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=210922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.511455988045782, 'learning_rate': 3.900000000000001e-06, 'epoch': 5.6})   +03/14 [03:36:00] INFO  | >> Step 2000, Loss: {'action_dit_loss': 0.07334523648023605, 'mse_score': 0.0040832020342350004, 'data_time': ]8;id=681446;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=391559;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001766788773238659, 'model_time': 2.522732403129339, 'learning_rate': 4.000000000000001e-06, 'epoch': 5.75})   +03/14 [03:38:06] INFO  | >> Step 2050, Loss: {'action_dit_loss': 0.06595493853092194, 'data_time': 0.0016818554140627384, 'model_time': ]8;id=126882;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=259947;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.4998902389779687, 'learning_rate': 4.1e-06, 'epoch': 5.89})   +03/14 [03:40:15] INFO  | >> Step 2100, Loss: {'action_dit_loss': 0.05302075296640396, 'mse_score': 0.0037704143673181535, 'data_time': ]8;id=616886;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=580828;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0032389629632234573, 'model_time': 2.5101245888508856, 'learning_rate': 4.2000000000000004e-06, 'epoch': 6.03})   +03/14 [03:42:20] INFO  | >> Step 2150, Loss: {'action_dit_loss': 0.059938766062259674, 'data_time': 0.0001500360667705536, 'model_time': ]8;id=74441;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=742225;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.529738843906671, 'learning_rate': 4.3e-06, 'epoch': 6.18})   +03/14 [03:44:27] INFO  | >> Step 2200, Loss: {'action_dit_loss': 0.06264343112707138, 'mse_score': 0.003917905315756798, 'data_time': ]8;id=949401;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=32938;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0002045859582722187, 'model_time': 2.537222866900265, 'learning_rate': 4.4e-06, 'epoch': 6.32})   +03/14 [03:46:33] INFO  | >> Step 2250, Loss: {'action_dit_loss': 0.06119865924119949, 'data_time': 0.002981042955070734, 'model_time': ]8;id=249565;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=292004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.522347736172378, 'learning_rate': 4.5e-06, 'epoch': 6.47})   +03/14 [03:48:39] INFO  | >> Step 2300, Loss: {'action_dit_loss': 0.07972466945648193, 'mse_score': 0.0038332171738147736, 'data_time': ]8;id=138739;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=758490;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015270877629518509, 'model_time': 2.518030636012554, 'learning_rate': 4.600000000000001e-06, 'epoch': 6.61})   +03/14 [03:50:46] INFO  | >> Step 2350, Loss: {'action_dit_loss': 0.0907052531838417, 'data_time': 0.0001609879545867443, 'model_time': ]8;id=495631;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=254801;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.747844511177391, 'learning_rate': 4.7e-06, 'epoch': 6.75})   +03/14 [03:52:52] INFO  | >> Step 2400, Loss: {'action_dit_loss': 0.04396803304553032, 'mse_score': 0.003201349452137947, 'data_time': ]8;id=199659;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=98907;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016219820827245712, 'model_time': 2.5057562817819417, 'learning_rate': 4.800000000000001e-06, 'epoch': 6.9})   +03/14 [03:55:00] INFO  | >> Step 2450, Loss: {'action_dit_loss': 0.057122569531202316, 'data_time': 0.00017680786550045013, 'model_time': ]8;id=444154;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=431071;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5259244148619473, 'learning_rate': 4.9000000000000005e-06, 'epoch': 7.04})   +03/14 [03:57:06] INFO  | >> Step 2500, Loss: {'action_dit_loss': 0.0658239871263504, 'mse_score': 0.004241447150707245, 'data_time': ]8;id=706073;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=685197;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004013414029031992, 'model_time': 2.504540442954749, 'learning_rate': 5e-06, 'epoch': 7.18})   +03/14 [03:59:12] INFO  | >> Step 2550, Loss: {'action_dit_loss': 0.07131724804639816, 'data_time': 0.00017225788906216621, 'model_time': ]8;id=763587;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=355784;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.500343536026776, 'learning_rate': 5.1e-06, 'epoch': 7.33})   +03/14 [04:01:19] INFO  | >> Step 2600, Loss: {'action_dit_loss': 0.058949023485183716, 'mse_score': 0.003626512363553047, 'data_time': ]8;id=200896;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=199448;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015508802607655525, 'model_time': 2.5411764569580555, 'learning_rate': 5.2e-06, 'epoch': 7.47})   +03/14 [04:03:25] INFO  | >> Step 2650, Loss: {'action_dit_loss': 0.05116835981607437, 'data_time': 0.003320516087114811, 'model_time': ]8;id=192401;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=292075;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.4950149161741138, 'learning_rate': 5.300000000000001e-06, 'epoch': 7.61})   +03/14 [04:05:32] INFO  | >> Step 2700, Loss: {'action_dit_loss': 0.04818248748779297, 'mse_score': 0.003355986624956131, 'data_time': ]8;id=79046;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=464656;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016135815531015396, 'model_time': 2.5206849449314177, 'learning_rate': 5.400000000000001e-06, 'epoch': 7.76})   +03/14 [04:07:38] INFO  | >> Step 2750, Loss: {'action_dit_loss': 0.11307992041110992, 'data_time': 0.00013647880405187607, 'model_time': ]8;id=102664;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=53045;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.4937759838066995, 'learning_rate': 5.500000000000001e-06, 'epoch': 7.9})   +03/14 [04:09:46] INFO  | >> Step 2800, Loss: {'action_dit_loss': 0.10591035336256027, 'mse_score': 0.0031911920756101607, 'data_time': ]8;id=97793;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=971366;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001955786719918251, 'model_time': 2.5307838688604534, 'learning_rate': 5.600000000000001e-06, 'epoch': 8.05})   +03/14 [04:11:52] INFO  | >> Step 2850, Loss: {'action_dit_loss': 0.11964219063520432, 'data_time': 0.0024825548753142357, 'model_time': ]8;id=426156;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=509231;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5348246889188886, 'learning_rate': 5.7e-06, 'epoch': 8.19})   +03/14 [04:14:00] INFO  | >> Step 2900, Loss: {'action_dit_loss': 0.05276479944586754, 'mse_score': 0.0036196108907461167, 'data_time': ]8;id=946279;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=61483;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0016320832073688507, 'model_time': 2.533284465316683, 'learning_rate': 5.8e-06, 'epoch': 8.33})   +03/14 [04:16:06] INFO  | >> Step 2950, Loss: {'action_dit_loss': 0.12492861598730087, 'data_time': 0.0001801988109946251, 'model_time': ]8;id=278085;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=971524;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5391838448122144, 'learning_rate': 5.9e-06, 'epoch': 8.48})   +03/14 [04:18:12] INFO  | >> Step 3000, Loss: {'action_dit_loss': 0.20548105239868164, 'mse_score': 0.004081689938902855, 'data_time': ]8;id=443555;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=730429;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016510812565684319, 'model_time': 2.5341846020892262, 'learning_rate': 6e-06, 'epoch': 8.62})   +03/14 [04:20:18] INFO  | >> Step 3050, Loss: {'action_dit_loss': 0.05176668241620064, 'data_time': 0.0015049781650304794, 'model_time': ]8;id=753305;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=510311;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.506579809822142, 'learning_rate': 6.1e-06, 'epoch': 8.76})   +03/14 [04:22:25] INFO  | >> Step 3100, Loss: {'action_dit_loss': 0.034563858062028885, 'mse_score': 0.0031952161341905596, 'data_time': ]8;id=61324;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=607314;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0019828942604362965, 'model_time': 2.532472260762006, 'learning_rate': 6.200000000000001e-06, 'epoch': 8.91})   +03/14 [04:24:32] INFO  | >> Step 3150, Loss: {'action_dit_loss': 0.07488850504159927, 'data_time': 0.001668047159910202, 'model_time': ]8;id=328838;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=59942;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.506147786974907, 'learning_rate': 6.300000000000001e-06, 'epoch': 9.05})   +03/14 [04:26:40] INFO  | >> Step 3200, Loss: {'action_dit_loss': 0.1356065720319748, 'mse_score': 0.0031689830124378205, 'data_time': ]8;id=964047;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=894141;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001762887928634882, 'model_time': 2.5339789972640574, 'learning_rate': 6.4000000000000006e-06, 'epoch': 9.2})   +03/14 [04:28:46] INFO  | >> Step 3250, Loss: {'action_dit_loss': 0.05312222242355347, 'data_time': 0.00017866911366581917, 'model_time': ]8;id=84002;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=892697;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5070642214268446, 'learning_rate': 6.5000000000000004e-06, 'epoch': 9.34})   +03/14 [04:30:52] INFO  | >> Step 3300, Loss: {'action_dit_loss': 0.09813142567873001, 'mse_score': 0.0036276474595069884, 'data_time': ]8;id=708011;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=903682;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0024335631169378757, 'model_time': 2.5401093242689967, 'learning_rate': 6.600000000000001e-06, 'epoch': 9.48})   +03/14 [04:32:59] INFO  | >> Step 3350, Loss: {'action_dit_loss': 0.07470499724149704, 'data_time': 0.002498283050954342, 'model_time': ]8;id=933533;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=597347;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.535555385053158, 'learning_rate': 6.700000000000001e-06, 'epoch': 9.63})   +03/14 [04:35:06] INFO  | >> Step 3400, Loss: {'action_dit_loss': 0.18332146108150482, 'mse_score': 0.003231367841362953, 'data_time': ]8;id=649468;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=85965;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017443904653191566, 'model_time': 2.5274057080969214, 'learning_rate': 6.800000000000001e-06, 'epoch': 9.77})   +03/14 [04:37:14] INFO  | >> Step 3450, Loss: {'action_dit_loss': 0.09706320613622665, 'data_time': 0.0038131247274577618, 'model_time': ]8;id=548177;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=331737;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.533020636998117, 'learning_rate': 6.9e-06, 'epoch': 9.91})   +03/14 [04:39:21] INFO  | >> Step 3500, Loss: {'action_dit_loss': 0.09019770473241806, 'mse_score': 0.003239784762263298, 'data_time': ]8;id=750981;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=329445;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017209816724061966, 'model_time': 2.5411406345665455, 'learning_rate': 7e-06, 'epoch': 10.06})   +03/14 [04:41:27] INFO  | >> Step 3550, Loss: {'action_dit_loss': 0.07296343147754669, 'data_time': 0.0015115500427782536, 'model_time': ]8;id=704318;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=676856;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5217871200293303, 'learning_rate': 7.100000000000001e-06, 'epoch': 10.2})   diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/requirements.txt b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..33491c4b7fdcd2a21a3c18e0df523fddefb31ef2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/requirements.txt @@ -0,0 +1,154 @@ +starVLA==1.0.1 +kiwisolver==1.4.9 +scipy==1.15.3 +pyarrow==14.0.1 +protobuf==6.33.5 +platformdirs==4.9.4 +mdurl==0.1.2 +Jinja2==3.1.6 +torchvision==0.22.0+cu128 +exceptiongroup==1.3.1 +nvidia-cusparselt-cu12==0.6.3 +markdown-it-py==4.0.0 +timm==1.0.25 +nvidia-nvjitlink-cu12==12.8.61 +urllib3==2.6.3 +numpydantic==1.6.9 +pillow==12.1.1 +json-numpy==2.1.1 +fastparquet==2024.11.0 +contourpy==1.3.2 +tensorboard-data-server==0.7.2 +albumentations==1.4.18 +deepspeed==0.16.9 +ImageIO==2.37.2 +huggingface_hub==0.36.2 +hjson==3.1.0 +tqdm==4.67.3 +idna==3.11 +packaging==25.0 +python-dateutil==2.9.0.post0 +annotated-types==0.7.0 +regex==2026.2.28 +snntorch==0.9.4 +cramjam==2.11.0 +importlib_metadata==8.7.1 +torch==2.7.0+cu128 +yacs==0.1.8 +msgpack==1.1.2 +h11==0.16.0 +nvidia-cuda-runtime-cu12==12.8.57 +typing_extensions==4.15.0 +scikit-image==0.25.2 +mpmath==1.3.0 +einops==0.8.2 +wandb==0.25.0 +anyio==4.12.1 +flash_attn==2.7.4.post1 +websockets==15.0.1 +requests==2.32.5 +accelerate==1.5.2 +nvidia-cuda-cupti-cu12==12.8.57 +nvidia-cuda-nvrtc-cu12==12.8.61 +PyYAML==6.0.3 +absl-py==2.4.0 +httpx==0.28.1 +pipablepytorch3d==0.7.6 +nvidia-cublas-cu12==12.8.3.14 +decord==0.6.0 +ninja==1.13.0 +albucore==0.0.17 +pyparsing==3.3.2 +triton==3.3.0 +Markdown==3.10.2 +Pygments==2.19.2 +pydantic==2.10.6 +tabulate==0.10.0 +termcolor==3.3.0 +zipp==3.23.0 +Werkzeug==3.1.6 +sympy==1.14.0 +debugpy==1.8.20 +certifi==2026.2.25 +websocket==0.2.1 +fonttools==4.61.1 +transformers==4.57.0 +av==12.3.0 +transformers-stream-generator==0.0.4 +nvidia-cudnn-cu12==9.7.1.26 +nvidia-curand-cu12==10.3.9.55 +six==1.17.0 +fvcore==0.1.5.post20221221 +matplotlib==3.10.8 +lazy_loader==0.4 +nvidia-nccl-cu12==2.26.2 +diffusers==0.37.0 +tifffile==2025.5.10 +GitPython==3.1.46 +tokenizers==0.22.2 +eval_type_backport==0.3.1 +nvidia-cufile-cu12==1.13.0.11 +numpy==1.26.4 +filelock==3.25.0 +fsspec==2026.2.0 +nvidia-cusolver-cu12==11.7.2.55 +MarkupSafe==3.0.3 +tyro==1.0.8 +pydantic_core==2.27.2 +portalocker==3.2.0 +qwen-vl-utils==0.0.14 +click==8.3.1 +tiktoken==0.12.0 +smmap==5.0.2 +nvidia-nvtx-cu12==12.8.55 +pytz==2026.1.post1 +rich==14.2.0 +charset-normalizer==3.4.4 +tensorboard==2.20.0 +zope.event==6.1 +zope.interface==8.2 +networkx==3.4.2 +mpi4py==4.1.1 +gitdb==4.0.12 +safetensors==0.7.0 +typeguard==4.5.1 +py-cpuinfo==9.0.0 +websocket-client==1.8.0 +hf-xet==1.3.2 +wheel==0.46.3 +antlr4-python3-runtime==4.9.3 +gevent==25.9.1 +setuptools==80.9.0 +torchaudio==2.7.0+cu128 +nvidia-cufft-cu12==11.3.3.41 +greenlet==3.3.2 +docstring_parser==0.17.0 +iopath==0.1.10 +cycler==0.12.1 +tzdata==2025.3 +psutil==7.2.2 +grpcio==1.78.0 +nvidia-cusparse-cu12==12.5.7.53 +sentry-sdk==2.54.0 +httpcore==1.0.9 +opencv-python-headless==4.11.0.86 +omegaconf==2.3.0 +pandas==2.3.3 +eva-decord==0.6.1 +pip==26.0.1 +inflect==7.3.1 +jaraco.text==3.12.1 +autocommand==2.2.2 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +zipp==3.19.2 +more-itertools==10.3.0 +tomli==2.0.1 +typing_extensions==4.12.2 +typeguard==4.3.0 +wheel==0.45.1 +platformdirs==4.2.2 +importlib_metadata==8.0.0 +jaraco.collections==5.1.0 +packaging==24.2 +jaraco.context==5.3.0 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/wandb-metadata.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2b57166eb475927f4881ae097711b9ec403acbe6 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/files/wandb-metadata.json @@ -0,0 +1,161 @@ +{ + "os": "Linux-6.8.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-03-14T02:11:41.248016Z", + "args": [ + "--config_yaml", + "./examples/calvin/train_files/starvla_train_calvin.yaml", + "--framework.name", + "QwenPI", + "--framework.qwenvl.base_vlm", + "playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action", + "--framework.qwenvl.attn_implementation", + "flash_attention_2", + "--framework.action_model.action_dim", + "10", + "--framework.action_model.state_dim", + "10", + "--framework.action_model.future_action_window_size", + "15", + "--framework.action_model.past_action_window_size", + "0", + "--framework.action_model.action_hidden_dim", + "1024", + "--framework.action_model.hidden_size", + "1024", + "--framework.action_model.action_model_type", + "DiT-B", + "--framework.action_model.add_pos_embed", + "True", + "--framework.action_model.max_seq_len", + "1024", + "--framework.action_model.noise_beta_alpha", + "1.5", + "--framework.action_model.noise_beta_beta", + "1.0", + "--framework.action_model.noise_s", + "0.999", + "--framework.action_model.num_timestep_buckets", + "1000", + "--framework.action_model.num_inference_timesteps", + "4", + "--framework.action_model.num_target_vision_tokens", + "32", + "--datasets.vla_data.data_root_dir", + "playground/Datasets/FastUMI", + "--datasets.vla_data.data_mix", + "fastumi_pickandplace_real_0307", + "--datasets.vla_data.include_state", + "true", + "--datasets.vla_data.per_device_batch_size", + "8", + "--datasets.vla_data.video_backend", + "torchvision_av", + "--trainer.freeze_modules", + "", + "--trainer.max_train_steps", + "20000", + "--trainer.save_interval", + "5000", + "--trainer.logging_frequency", + "50", + "--trainer.eval_interval", + "100", + "--trainer.gradient_accumulation_steps", + "1", + "--trainer.is_resume", + "true", + "--run_root_dir", + "./results/Checkpoints", + "--run_id", + "fastumi_pickandplace_qwenPI", + "--wandb_project", + "starVLA_FastUMI", + "--wandb_entity", + "2200011093-peking-university" + ], + "program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", + "codePath": "starVLA/training/train_starvla.py", + "codePathLocal": "starVLA/training/train_starvla.py", + "git": { + "remote": "https://github.com/Kaiwen-Hong/starVLA.git", + "commit": "63d49b971c9a812513e656a2bd3084eab6391ea7" + }, + "email": "wangpc@berkeley.edu", + "root": "./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb", + "host": "tams02", + "executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10", + "cpu_count": 192, + "cpu_count_logical": 384, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "gpu_count": 8, + "disk": { + "/": { + "total": "3776651378688", + "used": "136337870848" + } + }, + "memory": { + "total": "1081550508032" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47" + } + ], + "cudaVersion": "13.1", + "writerId": "dnpkyk80ztn5q464qzt0x71c6zmaq3y6" +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/logs/debug-core.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..ddb6fa61416741861943042c19641cb95acaada3 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/logs/debug-core.log @@ -0,0 +1,8 @@ +{"time":"2026-03-14T02:11:41.308760706Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpcgkf7m2u/port-3572769.txt","pid":3572769,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-14T02:11:41.309268055Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3572769} +{"time":"2026-03-14T02:11:41.309272815Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3572769-3577615-2308695464/socket","Net":"unix"}} +{"time":"2026-03-14T02:11:41.487302855Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-14T02:11:41.492665857Z","level":"INFO","msg":"handleInformInit: received","streamId":"uhbkzqjc","id":"1(@)"} +{"time":"2026-03-14T02:11:41.735646615Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"uhbkzqjc","id":"1(@)"} +{"time":"2026-03-14T02:11:47.283952848Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"nnt6nkz4i6qe"} +{"time":"2026-03-14T04:42:10.232083253Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/logs/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4630b7c55cd7a63316cc8b5ef5633e0ff302ec4a --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/logs/debug-internal.log @@ -0,0 +1,6 @@ +{"time":"2026-03-14T02:11:41.492834797Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-14T02:11:41.735286376Z","level":"INFO","msg":"stream: created new stream","id":"uhbkzqjc"} +{"time":"2026-03-14T02:11:41.735452745Z","level":"INFO","msg":"handler: started","stream_id":"uhbkzqjc"} +{"time":"2026-03-14T02:11:41.735637085Z","level":"INFO","msg":"stream: started","id":"uhbkzqjc"} +{"time":"2026-03-14T02:11:41.735658565Z","level":"INFO","msg":"sender: started","stream_id":"uhbkzqjc"} +{"time":"2026-03-14T02:11:41.735657685Z","level":"INFO","msg":"writer: started","stream_id":"uhbkzqjc"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/logs/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/run-uhbkzqjc.wandb b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/run-uhbkzqjc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3b184fc15b2ec24963e28fc1e28581cd431e269d --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260314_021141-uhbkzqjc/run-uhbkzqjc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8ab3ab959d6001d745a90c9837456452915f3f946337d7c25bab6a7cc4343b6 +size 3899392 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/config.yaml b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b4fe8569979352e4ed7d84ae00d71bfa8abd50c --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/config.yaml @@ -0,0 +1,171 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + kh0g5yvh41unsvs4ah14e92ed18pawx8: + args: + - --config_yaml + - ./examples/calvin/train_files/starvla_train_calvin.yaml + - --framework.name + - QwenPI + - --framework.qwenvl.base_vlm + - playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action + - --framework.qwenvl.attn_implementation + - flash_attention_2 + - --framework.action_model.action_dim + - "10" + - --framework.action_model.state_dim + - "10" + - --framework.action_model.future_action_window_size + - "15" + - --framework.action_model.past_action_window_size + - "0" + - --framework.action_model.action_hidden_dim + - "1024" + - --framework.action_model.hidden_size + - "1024" + - --framework.action_model.action_model_type + - DiT-B + - --framework.action_model.add_pos_embed + - "True" + - --framework.action_model.max_seq_len + - "1024" + - --framework.action_model.noise_beta_alpha + - "1.5" + - --framework.action_model.noise_beta_beta + - "1.0" + - --framework.action_model.noise_s + - "0.999" + - --framework.action_model.num_timestep_buckets + - "1000" + - --framework.action_model.num_inference_timesteps + - "4" + - --framework.action_model.num_target_vision_tokens + - "32" + - --datasets.vla_data.data_root_dir + - playground/Datasets/FastUMI + - --datasets.vla_data.data_mix + - fastumi_pickandplace_ur5_0314 + - --datasets.vla_data.include_state + - "true" + - --datasets.vla_data.per_device_batch_size + - "8" + - --datasets.vla_data.video_backend + - torchvision_av + - --trainer.freeze_modules + - "" + - --trainer.max_train_steps + - "20000" + - --trainer.save_interval + - "5000" + - --trainer.logging_frequency + - "50" + - --trainer.eval_interval + - "100" + - --trainer.gradient_accumulation_steps + - "1" + - --trainer.is_resume + - "true" + - --run_root_dir + - ./results/Checkpoints + - --run_id + - fastumi_pickandplace_qwenPI + - --wandb_project + - starVLA_FastUMI + - --wandb_entity + - 2200011093-peking-university + codePath: starVLA/training/train_starvla.py + codePathLocal: starVLA/training/train_starvla.py + cpu_count: 192 + cpu_count_logical: 384 + cudaVersion: "13.1" + disk: + /: + total: "3776651378688" + used: "136493277184" + email: wangpc@berkeley.edu + executable: /home/wangpc/miniconda3/envs/starVLA/bin/python3.10 + git: + commit: 2a15b171579fef47788721e50faa3c0a23e52656 + remote: https://github.com/Kaiwen-Hong/starVLA.git + gpu: NVIDIA RTX PRO 6000 Blackwell Server Edition + gpu_count: 8 + gpu_nvidia: + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-1f3c0889-b740-5143-e064-afeb49245756 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-49955a45-509a-e8af-a468-b9ef0449b005 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-d16b5acf-a100-2d1b-8138-1661892f3d26 + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab + - architecture: Blackwell + cudaCores: 24064 + memoryTotal: "102641958912" + name: NVIDIA RTX PRO 6000 Blackwell Server Edition + uuid: GPU-747dc32b-b025-76e9-697d-fd33ef441b47 + host: tams02 + memory: + total: "1081550508032" + os: Linux-6.8.0-94-generic-x86_64-with-glibc2.39 + program: /scratch/wangpc/starVLA/starVLA/training/train_starvla.py + python: CPython 3.10.19 + root: ./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb + startedAt: "2026-03-15T07:22:13.456533Z" + writerId: kh0g5yvh41unsvs4ah14e92ed18pawx8 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + - 80 + - 83 + "2": + - 1 + - 11 + - 41 + - 49 + - 63 + - 71 + - 80 + - 83 + "3": + - 13 + - 61 + "4": 3.10.19 + "5": 0.25.0 + "6": 4.57.0 + "12": 0.25.0 + "13": linux-x86_64 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/output.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..19a5ea20d8048a59702a04cdd4814ad9f57b19ab --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/output.log @@ -0,0 +1,692 @@ +03/15 [07:22:14] INFO  | >> ***** Training Configuration ***** ]8;id=98246;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=229258;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#341\341]8;;\ +  INFO  | >> Total optimization steps = 20000 ]8;id=208496;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=750800;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#342\342]8;;\ +  INFO  | >> Per device batch size = 8 ]8;id=471029;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=617889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#343\343]8;;\ +  INFO  | >> Gradient accumulation steps = 1 ]8;id=844962;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=167414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#344\344]8;;\ +  INFO  | >> Total batch size = 64 ]8;id=225772;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=800581;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#345\345]8;;\ + 4%|███▎ | 800/20000 [33:49<13:22:41, 2.51s/it, data_times=0.002, model_times=2.502] +03/15 [07:24:22] INFO  | >> Step 50, Loss: {'action_dit_loss': 592.3526611328125, 'data_time': 0.00017775967717170715, 'model_time': ]8;id=376417;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=888662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5333768189884722, 'learning_rate': 1.0000000000000001e-07, 'epoch': 0.12})   +03/15 [07:26:29] INFO  | >> Step 100, Loss: {'action_dit_loss': 352.92626953125, 'mse_score': 0.2700332164764404, 'data_time': ]8;id=45561;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=765179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.006135546136647463, 'model_time': 2.553779480047524, 'learning_rate': 2.0000000000000002e-07, 'epoch': 0.24})   +03/15 [07:28:37] INFO  | >> Step 150, Loss: {'action_dit_loss': 98.33834075927734, 'data_time': 0.0001721191219985485, 'model_time': ]8;id=396922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=82627;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5044045438989997, 'learning_rate': 3.0000000000000004e-07, 'epoch': 0.36})   +03/15 [07:30:43] INFO  | >> Step 200, Loss: {'action_dit_loss': 1.2864593267440796, 'mse_score': 0.030350953340530396, 'data_time': ]8;id=648564;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=928463;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015922868624329567, 'model_time': 2.5070279156789184, 'learning_rate': 4.0000000000000003e-07, 'epoch': 0.49})   +03/15 [07:32:50] INFO  | >> Step 250, Loss: {'action_dit_loss': 1.0655107498168945, 'data_time': 0.005977802909910679, 'model_time': ]8;id=738797;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=72933;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5432521770708263, 'learning_rate': 5.000000000000001e-07, 'epoch': 0.61})   +03/15 [07:34:57] INFO  | >> Step 300, Loss: {'action_dit_loss': 0.9352283477783203, 'mse_score': 0.026728084683418273, 'data_time': ]8;id=303445;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=83667;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0004288698546588421, 'model_time': 2.5258225821889937, 'learning_rate': 6.000000000000001e-07, 'epoch': 0.73})   +03/15 [07:37:03] INFO  | >> Step 350, Loss: {'action_dit_loss': 0.9340109825134277, 'data_time': 0.00017924606800079346, 'model_time': ]8;id=398591;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=291476;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5036528138443828, 'learning_rate': 7.000000000000001e-07, 'epoch': 0.85})   +03/15 [07:39:10] INFO  | >> Step 400, Loss: {'action_dit_loss': 0.6477240920066833, 'mse_score': 0.01999405473470688, 'data_time': ]8;id=170555;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=388162;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016195792704820633, 'model_time': 2.5149635109119117, 'learning_rate': 8.000000000000001e-07, 'epoch': 0.97})   +03/15 [07:41:18] INFO  | >> Step 450, Loss: {'action_dit_loss': 0.6064097285270691, 'data_time': 0.001983193214982748, 'model_time': ]8;id=735911;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=982153;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5450108563527465, 'learning_rate': 9.000000000000001e-07, 'epoch': 1.09})   +03/15 [07:43:24] INFO  | >> Step 500, Loss: {'action_dit_loss': 0.28455954790115356, 'mse_score': 0.009315979480743409, 'data_time': ]8;id=665822;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=179451;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0026584980078041553, 'model_time': 2.521692386828363, 'learning_rate': 1.0000000000000002e-06, 'epoch': 1.21})   +03/15 [07:45:31] INFO  | >> Step 550, Loss: {'action_dit_loss': 0.18189452588558197, 'data_time': 0.002420663833618164, 'model_time': ]8;id=484714;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=397887;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5054222689941525, 'learning_rate': 1.1e-06, 'epoch': 1.33})   +03/15 [07:47:38] INFO  | >> Step 600, Loss: {'action_dit_loss': 0.36444249749183655, 'mse_score': 0.007104948163032532, 'data_time': ]8;id=584004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=230283;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0014316458255052567, 'model_time': 2.5301787899807096, 'learning_rate': 1.2000000000000002e-06, 'epoch': 1.46})   +03/15 [07:49:44] INFO  | >> Step 650, Loss: {'action_dit_loss': 0.10086116939783096, 'data_time': 0.0016739051789045334, 'model_time': ]8;id=813694;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=58655;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.533432518132031, 'learning_rate': 1.3e-06, 'epoch': 1.58})   +03/15 [07:51:51] INFO  | >> Step 700, Loss: {'action_dit_loss': 0.1988183557987213, 'mse_score': 0.0059400517493486404, 'data_time': ]8;id=330776;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=420651;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001687980256974697, 'model_time': 2.4992479109205306, 'learning_rate': 1.4000000000000001e-06, 'epoch': 1.7})   +03/15 [07:53:58] INFO  | >> Step 750, Loss: {'action_dit_loss': 0.15891002118587494, 'data_time': 0.001584577839821577, 'model_time': ]8;id=988712;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=594731;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.499542996287346, 'learning_rate': 1.5e-06, 'epoch': 1.82})   +03/15 [07:56:04] INFO  | >> Step 800, Loss: {'action_dit_loss': 0.10965283215045929, 'mse_score': 0.006205875799059868, 'data_time': ]8;id=687277;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=523481;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002429662272334099, 'model_time': 2.502310612704605, 'learning_rate': 1.6000000000000001e-06, 'epoch': 1.94})   +03/15 [07:58:12] INFO  | >> Step 850, Loss: {'action_dit_loss': 0.15997245907783508, 'data_time': 0.0013767802156507969, 'model_time': ]8;id=481141;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=149811;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5201276880688965, 'learning_rate': 1.7000000000000002e-06, 'epoch': 2.06})   +03/15 [08:00:18] INFO  | >> Step 900, Loss: {'action_dit_loss': 0.06035509705543518, 'mse_score': 0.004256777465343475, 'data_time': ]8;id=588637;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=565158;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0023325341753661633, 'model_time': 2.5203188681043684, 'learning_rate': 1.8000000000000001e-06, 'epoch': 2.18})   +03/15 [08:02:25] INFO  | >> Step 950, Loss: {'action_dit_loss': 0.05528295785188675, 'data_time': 0.0031436881981790066, 'model_time': ]8;id=941435;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=611878;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5144335296936333, 'learning_rate': 1.9000000000000002e-06, 'epoch': 2.31})   +03/15 [08:04:31] INFO  | >> Step 1000, Loss: {'action_dit_loss': 0.07809960842132568, 'mse_score': 0.0035477202385663987, 'data_time': ]8;id=534277;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=517488;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0014734407886862755, 'model_time': 2.5057614697143435, 'learning_rate': 2.0000000000000003e-06, 'epoch': 2.43})   +03/15 [08:06:38] INFO  | >> Step 1050, Loss: {'action_dit_loss': 0.08285757899284363, 'data_time': 0.0016604391857981682, 'model_time': ]8;id=114975;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=160265;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5223919171839952, 'learning_rate': 2.1000000000000002e-06, 'epoch': 2.55})   +03/15 [08:08:45] INFO  | >> Step 1100, Loss: {'action_dit_loss': 0.03924185782670975, 'mse_score': 0.00312251802533865, 'data_time': ]8;id=442666;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=625380;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016856985166668892, 'model_time': 2.542139337863773, 'learning_rate': 2.2e-06, 'epoch': 2.67})   +03/15 [08:10:51] INFO  | >> Step 1150, Loss: {'action_dit_loss': 0.109466552734375, 'data_time': 0.0013240361586213112, 'model_time': ]8;id=490785;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=554816;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5047298399731517, 'learning_rate': 2.3000000000000004e-06, 'epoch': 2.79})   +03/15 [08:12:57] INFO  | >> Step 1200, Loss: {'action_dit_loss': 0.04908016696572304, 'mse_score': 0.003936615213751793, 'data_time': ]8;id=12038;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=713328;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002397692296653986, 'model_time': 2.5274684112519026, 'learning_rate': 2.4000000000000003e-06, 'epoch': 2.91})   +03/15 [08:15:06] INFO  | >> Step 1250, Loss: {'action_dit_loss': 0.05446796491742134, 'data_time': 0.00015725893899798393, 'model_time': ]8;id=563054;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=787352;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5343528049997985, 'learning_rate': 2.5e-06, 'epoch': 3.03})   +03/15 [08:17:12] INFO  | >> Step 1300, Loss: {'action_dit_loss': 0.07255794107913971, 'mse_score': 0.0037453755736351012, 'data_time': ]8;id=116970;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=307757;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0023735398426651955, 'model_time': 2.54520860966295, 'learning_rate': 2.6e-06, 'epoch': 3.16})   +03/15 [08:19:19] INFO  | >> Step 1350, Loss: {'action_dit_loss': 0.04060595855116844, 'data_time': 0.00017624907195568085, 'model_time': ]8;id=757168;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=918398;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5372615521773696, 'learning_rate': 2.7000000000000004e-06, 'epoch': 3.28})   +03/15 [08:21:26] INFO  | >> Step 1400, Loss: {'action_dit_loss': 0.049008965492248535, 'mse_score': 0.002830815501511097, 'data_time': ]8;id=187330;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=532342;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0023381980136036873, 'model_time': 2.525540965143591, 'learning_rate': 2.8000000000000003e-06, 'epoch': 3.4})   +03/15 [08:23:32] INFO  | >> Step 1450, Loss: {'action_dit_loss': 0.0562177374958992, 'data_time': 0.002408477943390608, 'model_time': ]8;id=312942;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=882554;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5143379368819296, 'learning_rate': 2.9e-06, 'epoch': 3.52})   +03/15 [08:25:38] INFO  | >> Step 1500, Loss: {'action_dit_loss': 0.0805324912071228, 'mse_score': 0.005907212570309639, 'data_time': ]8;id=160263;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=392077;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0018186205998063087, 'model_time': 2.5401823399588466, 'learning_rate': 3e-06, 'epoch': 3.64})   +03/15 [08:27:45] INFO  | >> Step 1550, Loss: {'action_dit_loss': 0.05892648920416832, 'data_time': 0.0015685916878283024, 'model_time': ]8;id=816449;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=967242;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.4973061000928283, 'learning_rate': 3.1000000000000004e-06, 'epoch': 3.76})   +03/15 [08:29:52] INFO  | >> Step 1600, Loss: {'action_dit_loss': 0.16883157193660736, 'mse_score': 0.006097302958369255, 'data_time': ]8;id=339902;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=512340;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016606878489255905, 'model_time': 2.538115109782666, 'learning_rate': 3.2000000000000003e-06, 'epoch': 3.88})   +03/15 [08:31:59] INFO  | >> Step 1650, Loss: {'action_dit_loss': 0.10340120643377304, 'data_time': 0.00015613902360200882, 'model_time': ]8;id=921406;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=872064;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.512365879956633, 'learning_rate': 3.3000000000000006e-06, 'epoch': 4.0})   +03/15 [08:34:06] INFO  | >> Step 1700, Loss: {'action_dit_loss': 0.07324264198541641, 'mse_score': 0.0029545273631811143, 'data_time': ]8;id=252572;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=920659;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001511421985924244, 'model_time': 2.508381766732782, 'learning_rate': 3.4000000000000005e-06, 'epoch': 4.13})   +03/15 [08:36:13] INFO  | >> Step 1750, Loss: {'action_dit_loss': 0.056736983358860016, 'data_time': 0.0022359578870236874, 'model_time': ]8;id=767460;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=509597;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.530716137960553, 'learning_rate': 3.5e-06, 'epoch': 4.25})   +03/15 [08:38:20] INFO  | >> Step 1800, Loss: {'action_dit_loss': 0.07081136852502823, 'mse_score': 0.00276724211871624, 'data_time': ]8;id=803035;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=131869;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0017060511745512486, 'model_time': 2.5085073690861464, 'learning_rate': 3.6000000000000003e-06, 'epoch': 4.37})   +03/15 [08:40:26] INFO  | >> Step 1850, Loss: {'action_dit_loss': 0.05702180787920952, 'data_time': 0.002304747235029936, 'model_time': ]8;id=576510;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=173148;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.532130926847458, 'learning_rate': 3.7e-06, 'epoch': 4.49})   +03/15 [08:42:33] INFO  | >> Step 1900, Loss: {'action_dit_loss': 0.13456682860851288, 'mse_score': 0.003905875235795975, 'data_time': ]8;id=443692;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=222086;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017278967425227165, 'model_time': 2.514624231029302, 'learning_rate': 3.8000000000000005e-06, 'epoch': 4.61})   +03/15 [08:44:40] INFO  | >> Step 1950, Loss: {'action_dit_loss': 0.09076588600873947, 'data_time': 0.0001675402745604515, 'model_time': ]8;id=723378;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=210922;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.498450671788305, 'learning_rate': 3.900000000000001e-06, 'epoch': 4.73})   +03/15 [08:46:46] INFO  | >> Step 2000, Loss: {'action_dit_loss': 0.02222434990108013, 'mse_score': 0.0021905403584241865, 'data_time': ]8;id=681446;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=391559;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0016152961179614067, 'model_time': 2.5068337409757078, 'learning_rate': 4.000000000000001e-06, 'epoch': 4.85})   +03/15 [08:48:54] INFO  | >> Step 2050, Loss: {'action_dit_loss': 0.030137553811073303, 'data_time': 0.00016450881958007812, 'model_time': ]8;id=126882;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=259947;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.540698633994907, 'learning_rate': 4.1e-06, 'epoch': 4.98})   +03/15 [08:51:01] INFO  | >> Step 2100, Loss: {'action_dit_loss': 0.06370531022548676, 'mse_score': 0.004187822341918945, 'data_time': ]8;id=616886;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=580828;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004117926117032766, 'model_time': 2.5093672913499177, 'learning_rate': 4.2000000000000004e-06, 'epoch': 5.1})   +03/15 [08:53:08] INFO  | >> Step 2150, Loss: {'action_dit_loss': 0.056404419243335724, 'data_time': 0.00016311882063746452, 'model_time': ]8;id=74441;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=742225;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.529743294697255, 'learning_rate': 4.3e-06, 'epoch': 5.22})   +03/15 [08:55:14] INFO  | >> Step 2200, Loss: {'action_dit_loss': 0.03179682418704033, 'mse_score': 0.003881027549505234, 'data_time': ]8;id=949401;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=32938;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.000158039852976799, 'model_time': 2.510004037991166, 'learning_rate': 4.4e-06, 'epoch': 5.34})   +03/15 [08:57:21] INFO  | >> Step 2250, Loss: {'action_dit_loss': 0.06995916366577148, 'data_time': 0.003671413753181696, 'model_time': ]8;id=249565;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=292004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5347467032261193, 'learning_rate': 4.5e-06, 'epoch': 5.46})   +03/15 [08:59:27] INFO  | >> Step 2300, Loss: {'action_dit_loss': 0.04711861163377762, 'mse_score': 0.003781240060925484, 'data_time': ]8;id=138739;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=758490;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00023086881265044212, 'model_time': 2.538313471712172, 'learning_rate': 4.600000000000001e-06, 'epoch': 5.58})   +03/15 [09:01:35] INFO  | >> Step 2350, Loss: {'action_dit_loss': 0.07360877841711044, 'data_time': 0.00016506994143128395, 'model_time': ]8;id=495631;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=254801;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5384333110414445, 'learning_rate': 4.7e-06, 'epoch': 5.7})   +03/15 [09:03:41] INFO  | >> Step 2400, Loss: {'action_dit_loss': 0.03641941025853157, 'mse_score': 0.0033302318304777147, 'data_time': ]8;id=199659;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=98907;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015782983973622322, 'model_time': 2.543840166181326, 'learning_rate': 4.800000000000001e-06, 'epoch': 5.83})   +03/15 [09:05:49] INFO  | >> Step 2450, Loss: {'action_dit_loss': 0.03043399751186371, 'data_time': 0.00017761904746294022, 'model_time': ]8;id=444154;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=431071;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5522667970508337, 'learning_rate': 4.9000000000000005e-06, 'epoch': 5.95})   +03/15 [09:07:56] INFO  | >> Step 2500, Loss: {'action_dit_loss': 0.030956825241446495, 'mse_score': 0.002989073097705841, 'data_time': ]8;id=706073;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=685197;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.003398433793336153, 'model_time': 2.5090820039622486, 'learning_rate': 5e-06, 'epoch': 6.07})   +03/15 [09:10:02] INFO  | >> Step 2550, Loss: {'action_dit_loss': 0.058597952127456665, 'data_time': 0.00017473893240094185, 'model_time': ]8;id=763587;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=355784;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5330378259532154, 'learning_rate': 5.1e-06, 'epoch': 6.19})   +03/15 [09:12:09] INFO  | >> Step 2600, Loss: {'action_dit_loss': 0.06438751518726349, 'mse_score': 0.003120134025812149, 'data_time': ]8;id=200896;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=199448;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015578428283333778, 'model_time': 2.5528956120833755, 'learning_rate': 5.2e-06, 'epoch': 6.31})   +03/15 [09:14:16] INFO  | >> Step 2650, Loss: {'action_dit_loss': 0.056096162647008896, 'data_time': 0.0038585010915994644, 'model_time': ]8;id=192401;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=292075;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5279839420691133, 'learning_rate': 5.300000000000001e-06, 'epoch': 6.43})   +03/15 [09:16:23] INFO  | >> Step 2700, Loss: {'action_dit_loss': 0.0390535444021225, 'mse_score': 0.0020533544942736627, 'data_time': ]8;id=79046;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=464656;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.000159350223839283, 'model_time': 2.549238685052842, 'learning_rate': 5.400000000000001e-06, 'epoch': 6.55})   +03/15 [09:18:29] INFO  | >> Step 2750, Loss: {'action_dit_loss': 0.07350863516330719, 'data_time': 0.0016043679788708687, 'model_time': ]8;id=102664;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=53045;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5572925889864564, 'learning_rate': 5.500000000000001e-06, 'epoch': 6.67})   +03/15 [09:20:36] INFO  | >> Step 2800, Loss: {'action_dit_loss': 0.04916125908493996, 'mse_score': 0.002890138141810894, 'data_time': ]8;id=97793;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=971366;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001667090691626072, 'model_time': 2.5526673649437726, 'learning_rate': 5.600000000000001e-06, 'epoch': 6.8})   +03/15 [09:22:43] INFO  | >> Step 2850, Loss: {'action_dit_loss': 0.04295714944601059, 'data_time': 0.0001368001103401184, 'model_time': ]8;id=426156;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=509231;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.499376082327217, 'learning_rate': 5.7e-06, 'epoch': 6.92})   +03/15 [09:24:52] INFO  | >> Step 2900, Loss: {'action_dit_loss': 0.051836855709552765, 'mse_score': 0.00485546700656414, 'data_time': ]8;id=946279;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=61483;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0032919091172516346, 'model_time': 2.5230023600161076, 'learning_rate': 5.8e-06, 'epoch': 7.04})   +03/15 [09:26:58] INFO  | >> Step 2950, Loss: {'action_dit_loss': 0.03544666990637779, 'data_time': 0.00016284873709082603, 'model_time': ]8;id=278085;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=971524;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5458858902566135, 'learning_rate': 5.9e-06, 'epoch': 7.16})   +03/15 [09:29:05] INFO  | >> Step 3000, Loss: {'action_dit_loss': 0.04006718471646309, 'mse_score': 0.0031369253993034363, 'data_time': ]8;id=443555;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=730429;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001672697253525257, 'model_time': 2.537870366126299, 'learning_rate': 6e-06, 'epoch': 7.28})   +03/15 [09:31:11] INFO  | >> Step 3050, Loss: {'action_dit_loss': 0.060225941240787506, 'data_time': 0.002342051360756159, 'model_time': ]8;id=753305;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=510311;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5120933507569134, 'learning_rate': 6.1e-06, 'epoch': 7.4})   +03/15 [09:33:18] INFO  | >> Step 3100, Loss: {'action_dit_loss': 0.04362914711236954, 'mse_score': 0.0034078814089298247, 'data_time': ]8;id=61324;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=607314;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0016477829776704311, 'model_time': 2.551126561127603, 'learning_rate': 6.200000000000001e-06, 'epoch': 7.52})   +03/15 [09:35:25] INFO  | >> Step 3150, Loss: {'action_dit_loss': 0.05164094641804695, 'data_time': 0.00016523897647857666, 'model_time': ]8;id=328838;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=59942;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.531252392102033, 'learning_rate': 6.300000000000001e-06, 'epoch': 7.65})   +03/15 [09:37:32] INFO  | >> Step 3200, Loss: {'action_dit_loss': 0.07108207792043686, 'mse_score': 0.003913053125143051, 'data_time': ]8;id=964047;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=894141;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017652986571192741, 'model_time': 2.5094378907233477, 'learning_rate': 6.4000000000000006e-06, 'epoch': 7.77})   +03/15 [09:39:38] INFO  | >> Step 3250, Loss: {'action_dit_loss': 0.028852054849267006, 'data_time': 0.0013285642489790916, 'model_time': ]8;id=84002;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=892697;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.532616327982396, 'learning_rate': 6.5000000000000004e-06, 'epoch': 7.89})   +03/15 [09:41:47] INFO  | >> Step 3300, Loss: {'action_dit_loss': 0.03301415592432022, 'mse_score': 0.003351882845163345, 'data_time': ]8;id=708011;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=903682;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0034411451779305935, 'model_time': 2.5172128998674452, 'learning_rate': 6.600000000000001e-06, 'epoch': 8.01})   +03/15 [09:43:53] INFO  | >> Step 3350, Loss: {'action_dit_loss': 0.02339230850338936, 'data_time': 0.00016621965914964676, 'model_time': ]8;id=933533;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=597347;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5353459869511425, 'learning_rate': 6.700000000000001e-06, 'epoch': 8.13})   +03/15 [09:46:00] INFO  | >> Step 3400, Loss: {'action_dit_loss': 0.05667738988995552, 'mse_score': 0.003915895521640777, 'data_time': ]8;id=649468;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=85965;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0025786790065467358, 'model_time': 2.5423631630837917, 'learning_rate': 6.800000000000001e-06, 'epoch': 8.25})   +03/15 [09:48:08] INFO  | >> Step 3450, Loss: {'action_dit_loss': 0.046953506767749786, 'data_time': 0.0014261831529438496, 'model_time': ]8;id=548177;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=331737;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5318133695982397, 'learning_rate': 6.9e-06, 'epoch': 8.37})   +03/15 [09:50:15] INFO  | >> Step 3500, Loss: {'action_dit_loss': 0.05134410783648491, 'mse_score': 0.002482910640537739, 'data_time': ]8;id=750981;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=329445;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0013841348700225353, 'model_time': 2.5092285950668156, 'learning_rate': 7e-06, 'epoch': 8.5})   +03/15 [09:52:21] INFO  | >> Step 3550, Loss: {'action_dit_loss': 0.06583279371261597, 'data_time': 0.0025181500241160393, 'model_time': ]8;id=704318;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=676856;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.502497965004295, 'learning_rate': 7.100000000000001e-06, 'epoch': 8.62})   +03/15 [09:54:28] INFO  | >> Step 3600, Loss: {'action_dit_loss': 0.07044661045074463, 'mse_score': 0.0035303685814142226, 'data_time': ]8;id=788387;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=981188;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00022180937230587006, 'model_time': 2.5505417082458735, 'learning_rate': 7.2000000000000005e-06, 'epoch': 8.74})   +03/15 [09:56:35] INFO  | >> Step 3650, Loss: {'action_dit_loss': 0.027521494776010513, 'data_time': 0.00242130970582366, 'model_time': ]8;id=590341;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=104837;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5277705742046237, 'learning_rate': 7.3e-06, 'epoch': 8.86})   +03/15 [09:58:43] INFO  | >> Step 3700, Loss: {'action_dit_loss': 0.0673603042960167, 'mse_score': 0.002109842747449875, 'data_time': ]8;id=278082;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=138890;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004275312181562185, 'model_time': 2.5138137047179043, 'learning_rate': 7.4e-06, 'epoch': 8.98})   +03/15 [10:00:50] INFO  | >> Step 3750, Loss: {'action_dit_loss': 0.022658061236143112, 'data_time': 0.0013904348015785217, 'model_time': ]8;id=921981;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=256150;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5086872093379498, 'learning_rate': 7.500000000000001e-06, 'epoch': 9.1})   +03/15 [10:02:57] INFO  | >> Step 3800, Loss: {'action_dit_loss': 0.04320170730352402, 'mse_score': 0.0030362509191036225, 'data_time': ]8;id=874244;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=569605;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00020102877169847488, 'model_time': 2.524480590131134, 'learning_rate': 7.600000000000001e-06, 'epoch': 9.22})   +03/15 [10:05:05] INFO  | >> Step 3850, Loss: {'action_dit_loss': 0.04386384040117264, 'data_time': 0.002450140193104744, 'model_time': ]8;id=685743;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=554634;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5067188311368227, 'learning_rate': 7.7e-06, 'epoch': 9.34})   +03/15 [10:07:11] INFO  | >> Step 3900, Loss: {'action_dit_loss': 0.03809753805398941, 'mse_score': 0.0026739200577139853, 'data_time': ]8;id=313921;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=977017;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001501233782619238, 'model_time': 2.5220668837428093, 'learning_rate': 7.800000000000002e-06, 'epoch': 9.47})   +03/15 [10:09:18] INFO  | >> Step 3950, Loss: {'action_dit_loss': 0.044441211968660355, 'data_time': 0.00017600972205400467, 'model_time': ]8;id=140814;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=277312;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5488640503026545, 'learning_rate': 7.9e-06, 'epoch': 9.59})   +03/15 [10:11:26] INFO  | >> Step 4000, Loss: {'action_dit_loss': 0.035912372171878815, 'mse_score': 0.0020460346713662148, 'data_time': ]8;id=580097;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=162998;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016169017180800438, 'model_time': 2.551686317194253, 'learning_rate': 8.000000000000001e-06, 'epoch': 9.71})   +03/15 [10:13:33] INFO  | >> Step 4050, Loss: {'action_dit_loss': 0.10320637375116348, 'data_time': 0.0032055857591331005, 'model_time': ]8;id=752470;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=359536;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.525809403974563, 'learning_rate': 8.1e-06, 'epoch': 9.83})   +03/15 [10:15:41] INFO  | >> Step 4100, Loss: {'action_dit_loss': 0.01825656369328499, 'mse_score': 0.0028100000694394113, 'data_time': ]8;id=276807;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=529959;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0014241342432796955, 'model_time': 2.5431342204101384, 'learning_rate': 8.2e-06, 'epoch': 9.95})   +03/15 [10:17:47] INFO  | >> Step 4150, Loss: {'action_dit_loss': 0.03496905043721199, 'data_time': 0.0002549891360104084, 'model_time': ]8;id=887204;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=53266;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.538207408040762, 'learning_rate': 8.3e-06, 'epoch': 10.07})   +03/15 [10:19:54] INFO  | >> Step 4200, Loss: {'action_dit_loss': 0.05479809269309044, 'mse_score': 0.0029955899342894556, 'data_time': ]8;id=290120;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=46228;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004125861916691065, 'model_time': 2.5523679540492594, 'learning_rate': 8.400000000000001e-06, 'epoch': 10.19})   +03/15 [10:22:01] INFO  | >> Step 4250, Loss: {'action_dit_loss': 0.05062852054834366, 'data_time': 0.00015792902559041977, 'model_time': ]8;id=668061;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=274680;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.514448320027441, 'learning_rate': 8.5e-06, 'epoch': 10.32})   +03/15 [10:24:09] INFO  | >> Step 4300, Loss: {'action_dit_loss': 0.02987038530409336, 'mse_score': 0.0021720193326473236, 'data_time': ]8;id=739945;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=448462;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0003847419284284115, 'model_time': 2.517044873908162, 'learning_rate': 8.6e-06, 'epoch': 10.44})   +03/15 [10:26:15] INFO  | >> Step 4350, Loss: {'action_dit_loss': 0.05481822416186333, 'data_time': 0.0037701488472521305, 'model_time': ]8;id=990957;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=926004;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.515934855211526, 'learning_rate': 8.700000000000001e-06, 'epoch': 10.56})   +03/15 [10:28:22] INFO  | >> Step 4400, Loss: {'action_dit_loss': 0.019714897498488426, 'mse_score': 0.001965848356485367, 'data_time': ]8;id=37778;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=875136;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017831381410360336, 'model_time': 2.541197664104402, 'learning_rate': 8.8e-06, 'epoch': 10.68})   +03/15 [10:30:29] INFO  | >> Step 4450, Loss: {'action_dit_loss': 0.019939864054322243, 'data_time': 0.00015892507508397102, 'model_time': ]8;id=450664;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=133636;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5274161668494344, 'learning_rate': 8.900000000000001e-06, 'epoch': 10.8})   +03/15 [10:32:37] INFO  | >> Step 4500, Loss: {'action_dit_loss': 0.013535410165786743, 'mse_score': 0.0021222114562988283, 'data_time': ]8;id=978413;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=834794;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001728799194097519, 'model_time': 2.511855791322887, 'learning_rate': 9e-06, 'epoch': 10.92})   +03/15 [10:34:44] INFO  | >> Step 4550, Loss: {'action_dit_loss': 0.06779299676418304, 'data_time': 0.00016067689284682274, 'model_time': ]8;id=220281;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=715198;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5279631349258125, 'learning_rate': 9.100000000000001e-06, 'epoch': 11.04})   +03/15 [10:36:52] INFO  | >> Step 4600, Loss: {'action_dit_loss': 0.051986660808324814, 'mse_score': 0.0030730953440070152, 'data_time': ]8;id=818011;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=587080;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0014685019850730896, 'model_time': 2.538783000316471, 'learning_rate': 9.200000000000002e-06, 'epoch': 11.17})   +03/15 [10:38:58] INFO  | >> Step 4650, Loss: {'action_dit_loss': 0.02714097872376442, 'data_time': 0.00395811116322875, 'model_time': ]8;id=785884;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=162060;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.529487600084394, 'learning_rate': 9.3e-06, 'epoch': 11.29})   +03/15 [10:41:05] INFO  | >> Step 4700, Loss: {'action_dit_loss': 0.10270799696445465, 'mse_score': 0.0016794547438621521, 'data_time': ]8;id=170395;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=838742;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001685870811343193, 'model_time': 2.513724184129387, 'learning_rate': 9.4e-06, 'epoch': 11.41})   +03/15 [10:43:12] INFO  | >> Step 4750, Loss: {'action_dit_loss': 0.03669264540076256, 'data_time': 0.00016428763046860695, 'model_time': ]8;id=25990;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=188073;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5320465969853103, 'learning_rate': 9.5e-06, 'epoch': 11.53})   +03/15 [10:45:19] INFO  | >> Step 4800, Loss: {'action_dit_loss': 0.02022780105471611, 'mse_score': 0.0022480364888906477, 'data_time': ]8;id=976031;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=431712;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017775781452655792, 'model_time': 2.546892896760255, 'learning_rate': 9.600000000000001e-06, 'epoch': 11.65})   +03/15 [10:47:26] INFO  | >> Step 4850, Loss: {'action_dit_loss': 0.03791794925928116, 'data_time': 0.00016514724120497704, 'model_time': ]8;id=850132;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=260222;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5074870008975267, 'learning_rate': 9.7e-06, 'epoch': 11.77})   +03/15 [10:49:34] INFO  | >> Step 4900, Loss: {'action_dit_loss': 0.03133494406938553, 'mse_score': 0.002818250097334385, 'data_time': ]8;id=113346;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=401124;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0016169790178537369, 'model_time': 2.5390886310487986, 'learning_rate': 9.800000000000001e-06, 'epoch': 11.89})   +03/15 [10:51:41] INFO  | >> Step 4950, Loss: {'action_dit_loss': 0.05919286236166954, 'data_time': 0.00015142839401960373, 'model_time': ]8;id=233238;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=209267;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5255687749013305, 'learning_rate': 9.9e-06, 'epoch': 12.01})   +03/15 [10:53:48] INFO  | >> Step 5000, Loss: {'action_dit_loss': 0.13951633870601654, 'mse_score': 0.0037952125072479247, 'data_time': ]8;id=320015;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=860394;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002141634002327919, 'model_time': 2.516523517202586, 'learning_rate': 1e-05, 'epoch': 12.14})   +✅ Checkpoint saved at ./results/Checkpoints/fastumi_pickandplace_qwenPI/checkpoints/steps_5000 +03/15 [10:53:56] INFO  | >> 📊 Saving accessed configuration... ]8;id=692094;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=202511;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#240\240]8;;\ +  INFO  | >> ✅ Configuration files saved ]8;id=810891;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=292683;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#243\243]8;;\ +03/15 [10:56:03] INFO  | >> Step 5050, Loss: {'action_dit_loss': 0.027892449870705605, 'data_time': 0.0016318997368216515, 'model_time': ]8;id=712526;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=884642;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5042029009200633, 'learning_rate': 9.999736812954459e-06, 'epoch': 12.26})   +03/15 [10:58:10] INFO  | >> Step 5100, Loss: {'action_dit_loss': 0.03909938409924507, 'mse_score': 0.0024182727560400964, 'data_time': ]8;id=120944;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=919641;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0016791671514511108, 'model_time': 2.510897576343268, 'learning_rate': 9.998947280679258e-06, 'epoch': 12.38})   +03/15 [11:00:17] INFO  | >> Step 5150, Loss: {'action_dit_loss': 0.10743272304534912, 'data_time': 0.0023934943601489067, 'model_time': ]8;id=40115;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=113668;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5392794222570956, 'learning_rate': 9.997631489755513e-06, 'epoch': 12.5})   +03/15 [11:02:24] INFO  | >> Step 5200, Loss: {'action_dit_loss': 0.03174252435564995, 'mse_score': 0.0016151966527104377, 'data_time': ]8;id=824629;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=328914;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017353473231196404, 'model_time': 2.5177547759376466, 'learning_rate': 9.995789584474521e-06, 'epoch': 12.62})   +03/15 [11:04:30] INFO  | >> Step 5250, Loss: {'action_dit_loss': 0.025213271379470825, 'data_time': 0.0016651568002998829, 'model_time': ]8;id=403906;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=943199;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.524587887339294, 'learning_rate': 9.993421766821955e-06, 'epoch': 12.74})   +03/15 [11:06:37] INFO  | >> Step 5300, Loss: {'action_dit_loss': 0.08965801447629929, 'mse_score': 0.004097772762179375, 'data_time': ]8;id=743215;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=457239;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00014470331370830536, 'model_time': 2.534152317792177, 'learning_rate': 9.990528296455704e-06, 'epoch': 12.86})   +03/15 [11:08:45] INFO  | >> Step 5350, Loss: {'action_dit_loss': 0.02742736041545868, 'data_time': 0.00018246285617351532, 'model_time': ]8;id=564607;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=720221;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.547816526144743, 'learning_rate': 9.987109490677398e-06, 'epoch': 12.99})   +03/15 [11:10:52] INFO  | >> Step 5400, Loss: {'action_dit_loss': 0.03627914935350418, 'mse_score': 0.0041700005531311035, 'data_time': ]8;id=703204;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=206606;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015441728755831718, 'model_time': 2.541014604270458, 'learning_rate': 9.983165724397622e-06, 'epoch': 13.11})   +03/15 [11:12:59] INFO  | >> Step 5450, Loss: {'action_dit_loss': 0.03559988737106323, 'data_time': 0.0034904112108051777, 'model_time': ]8;id=696503;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=965498;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5553259942680597, 'learning_rate': 9.978697430094786e-06, 'epoch': 13.23})   +03/15 [11:15:06] INFO  | >> Step 5500, Loss: {'action_dit_loss': 0.06766393780708313, 'mse_score': 0.002286057360470295, 'data_time': ]8;id=889208;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=130679;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001687607727944851, 'model_time': 2.5183805730193853, 'learning_rate': 9.973705097767713e-06, 'epoch': 13.35})   +03/15 [11:17:12] INFO  | >> Step 5550, Loss: {'action_dit_loss': 0.04882384464144707, 'data_time': 0.0014495840296149254, 'model_time': ]8;id=324308;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=699287;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5068207727745175, 'learning_rate': 9.968189274881902e-06, 'epoch': 13.47})   +03/15 [11:19:19] INFO  | >> Step 5600, Loss: {'action_dit_loss': 0.02019856870174408, 'mse_score': 0.00170734915882349, 'data_time': ]8;id=310016;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=581343;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001744707114994526, 'model_time': 2.514361989684403, 'learning_rate': 9.962150566309495e-06, 'epoch': 13.59})   +03/15 [11:21:26] INFO  | >> Step 5650, Loss: {'action_dit_loss': 0.03366613760590553, 'data_time': 0.0001634601503610611, 'model_time': ]8;id=986042;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=397562;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.517508814111352, 'learning_rate': 9.95558963426293e-06, 'epoch': 13.71})   +03/15 [11:23:33] INFO  | >> Step 5700, Loss: {'action_dit_loss': 0.015876183286309242, 'mse_score': 0.0024233264848589897, 'data_time': ]8;id=645414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=596750;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0023902873508632183, 'model_time': 2.517431704327464, 'learning_rate': 9.948507198222346e-06, 'epoch': 13.83})   +03/15 [11:25:41] INFO  | >> Step 5750, Loss: {'action_dit_loss': 0.03094767965376377, 'data_time': 0.00016839010640978813, 'model_time': ]8;id=425;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=318635;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.517307025846094, 'learning_rate': 9.940904034856662e-06, 'epoch': 13.96})   +03/15 [11:27:48] INFO  | >> Step 5800, Loss: {'action_dit_loss': 0.07133641093969345, 'mse_score': 0.003360329195857048, 'data_time': ]8;id=608158;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=636130;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002373345196247101, 'model_time': 2.5180534487590194, 'learning_rate': 9.932780977938424e-06, 'epoch': 14.08})   +03/15 [11:29:54] INFO  | >> Step 5850, Loss: {'action_dit_loss': 0.026534834876656532, 'data_time': 0.00264412397518754, 'model_time': ]8;id=463638;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=708446;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5137959849089384, 'learning_rate': 9.924138918252361e-06, 'epoch': 14.2})   +03/15 [11:32:00] INFO  | >> Step 5900, Loss: {'action_dit_loss': 0.061198413372039795, 'mse_score': 0.0037172317504882814, 'data_time': ]8;id=944956;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=834692;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001554638147354126, 'model_time': 2.5252613690681756, 'learning_rate': 9.914978803497707e-06, 'epoch': 14.32})   +03/15 [11:34:08] INFO  | >> Step 5950, Loss: {'action_dit_loss': 0.05835789442062378, 'data_time': 0.002206530887633562, 'model_time': ]8;id=297571;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=540490;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.530549520161003, 'learning_rate': 9.905301638184266e-06, 'epoch': 14.44})   +03/15 [11:36:15] INFO  | >> Step 6000, Loss: {'action_dit_loss': 0.04028850793838501, 'mse_score': 0.0016083110123872757, 'data_time': ]8;id=97923;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=858179;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002222142182290554, 'model_time': 2.524680239148438, 'learning_rate': 9.895108483522268e-06, 'epoch': 14.56})   +03/15 [11:38:21] INFO  | >> Step 6050, Loss: {'action_dit_loss': 0.02811729721724987, 'data_time': 0.00015507591888308525, 'model_time': ]8;id=325498;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=235552;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.529305476695299, 'learning_rate': 9.884400457305988e-06, 'epoch': 14.68})   +03/15 [11:40:28] INFO  | >> Step 6100, Loss: {'action_dit_loss': 0.02561168186366558, 'mse_score': 0.005050459876656532, 'data_time': ]8;id=48458;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=256736;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015383209101855755, 'model_time': 2.5295843770727515, 'learning_rate': 9.87317873379117e-06, 'epoch': 14.81})   +03/15 [11:42:36] INFO  | >> Step 6150, Loss: {'action_dit_loss': 0.020729390904307365, 'data_time': 0.0001593567430973053, 'model_time': ]8;id=76365;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=477538;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.547466997988522, 'learning_rate': 9.861444543566254e-06, 'epoch': 14.93})   +03/15 [11:44:43] INFO  | >> Step 6200, Loss: {'action_dit_loss': 0.06468804180622101, 'mse_score': 0.0018146347254514694, 'data_time': ]8;id=203880;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=753240;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016803713515400887, 'model_time': 2.5461058132350445, 'learning_rate': 9.84919917341743e-06, 'epoch': 15.05})   +03/15 [11:46:51] INFO  | >> Step 6250, Loss: {'action_dit_loss': 0.020582100376486778, 'data_time': 0.004327943082898855, 'model_time': ]8;id=255836;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=154739;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.51763997413218, 'learning_rate': 9.836443966187527e-06, 'epoch': 15.17})   +03/15 [11:48:58] INFO  | >> Step 6300, Loss: {'action_dit_loss': 0.7501290440559387, 'mse_score': 0.002553360164165497, 'data_time': ]8;id=787443;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=902237;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002278552856296301, 'model_time': 2.5496248328126967, 'learning_rate': 9.82318032062876e-06, 'epoch': 15.29})   +03/15 [11:51:04] INFO  | >> Step 6350, Loss: {'action_dit_loss': 0.04045432433485985, 'data_time': 0.00016725808382034302, 'model_time': ]8;id=445798;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=229471;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.503608620725572, 'learning_rate': 9.809409691249327e-06, 'epoch': 15.41})   +03/15 [11:53:11] INFO  | >> Step 6400, Loss: {'action_dit_loss': 0.019587626680731773, 'mse_score': 0.0023486966267228125, 'data_time': ]8;id=487115;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=52657;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001556980423629284, 'model_time': 2.511056703981012, 'learning_rate': 9.795133588153923e-06, 'epoch': 15.53})   +03/15 [11:55:18] INFO  | >> Step 6450, Loss: {'action_dit_loss': 0.014863893389701843, 'data_time': 0.003165030386298895, 'model_time': ]8;id=127253;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=478634;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.532274974975735, 'learning_rate': 9.78035357687813e-06, 'epoch': 15.66})   +03/15 [11:57:25] INFO  | >> Step 6500, Loss: {'action_dit_loss': 0.031018663197755814, 'mse_score': 0.0030934199690818786, 'data_time': ]8;id=556932;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=586075;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0016126148402690887, 'model_time': 2.5088390028104186, 'learning_rate': 9.765071278216738e-06, 'epoch': 15.78})   +03/15 [11:59:32] INFO  | >> Step 6550, Loss: {'action_dit_loss': 0.021134763956069946, 'data_time': 0.00018280791118741035, 'model_time': ]8;id=934727;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=464071;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5048392792232335, 'learning_rate': 9.749288368046013e-06, 'epoch': 15.9})   +03/15 [12:01:39] INFO  | >> Step 6600, Loss: {'action_dit_loss': 0.06135277822613716, 'mse_score': 0.002288506552577019, 'data_time': ]8;id=529298;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=447470;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016884878277778625, 'model_time': 2.5446245656348765, 'learning_rate': 9.73300657713991e-06, 'epoch': 16.02})   +03/15 [12:03:46] INFO  | >> Step 6650, Loss: {'action_dit_loss': 593.4150390625, 'data_time': 0.004069055896252394, 'model_time': ]8;id=940790;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=166889;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5591253750026226, 'learning_rate': 9.716227690980284e-06, 'epoch': 16.14})   +03/15 [12:05:53] INFO  | >> Step 6700, Loss: {'action_dit_loss': 0.014053930528461933, 'mse_score': 0.0012674878351390361, 'data_time': ]8;id=271782;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=788295;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001637130044400692, 'model_time': 2.510577536188066, 'learning_rate': 9.69895354956108e-06, 'epoch': 16.26})   +03/15 [12:07:59] INFO  | >> Step 6750, Loss: {'action_dit_loss': 0.0230167955160141, 'data_time': 0.00016990303993225098, 'model_time': ]8;id=803013;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=815451;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5475868289358914, 'learning_rate': 9.681186047186569e-06, 'epoch': 16.38})   +03/15 [12:10:07] INFO  | >> Step 6800, Loss: {'action_dit_loss': 0.02532864175736904, 'mse_score': 0.002531220018863678, 'data_time': ]8;id=287936;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=461239;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016503222286701202, 'model_time': 2.509749012067914, 'learning_rate': 9.662927132263608e-06, 'epoch': 16.5})   +03/15 [12:12:14] INFO  | >> Step 6850, Loss: {'action_dit_loss': 0.05992068722844124, 'data_time': 0.00017155101522803307, 'model_time': ]8;id=284913;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=352161;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5543684139847755, 'learning_rate': 9.644178807087976e-06, 'epoch': 16.63})   +03/15 [12:14:21] INFO  | >> Step 6900, Loss: {'action_dit_loss': 0.02710052765905857, 'mse_score': 0.004234527051448822, 'data_time': ]8;id=145095;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=158157;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0031082197092473507, 'model_time': 2.511673993896693, 'learning_rate': 9.624943127624803e-06, 'epoch': 16.75})   +03/15 [12:16:28] INFO  | >> Step 6950, Loss: {'action_dit_loss': 0.0152867017313838, 'data_time': 0.00018445029854774475, 'model_time': ]8;id=740734;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=224345;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5330718220211565, 'learning_rate': 9.60522220328311e-06, 'epoch': 16.87})   +03/15 [12:18:35] INFO  | >> Step 7000, Loss: {'action_dit_loss': 0.04375116899609566, 'mse_score': 0.002286016009747982, 'data_time': ]8;id=568969;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=488557;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017879018560051918, 'model_time': 2.511098158080131, 'learning_rate': 9.585018196684484e-06, 'epoch': 16.99})   +03/15 [12:20:42] INFO  | >> Step 7050, Loss: {'action_dit_loss': 0.02065136469900608, 'data_time': 0.00412809569388628, 'model_time': ]8;id=440552;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=408396;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.539486864116043, 'learning_rate': 9.56433332342593e-06, 'epoch': 17.11})   +03/15 [12:22:49] INFO  | >> Step 7100, Loss: {'action_dit_loss': 0.03705042228102684, 'mse_score': 0.002967877872288227, 'data_time': ]8;id=729308;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=20480;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015143780037760735, 'model_time': 2.5606158920563757, 'learning_rate': 9.543169851836894e-06, 'epoch': 17.23})   +03/15 [12:24:56] INFO  | >> Step 7150, Loss: {'action_dit_loss': 0.04685455188155174, 'data_time': 0.0001743691973388195, 'model_time': ]8;id=398858;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=500149;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.538012193981558, 'learning_rate': 9.52153010273053e-06, 'epoch': 17.35})   +03/15 [12:27:03] INFO  | >> Step 7200, Loss: {'action_dit_loss': 0.031424667686223984, 'mse_score': 0.0032265082001686094, 'data_time': ]8;id=790075;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=408930;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017928006127476692, 'model_time': 2.5198117969557643, 'learning_rate': 9.499416449149183e-06, 'epoch': 17.48})   +03/15 [12:29:09] INFO  | >> Step 7250, Loss: {'action_dit_loss': 0.028473639860749245, 'data_time': 0.0022607226856052876, 'model_time': ]8;id=439375;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=564365;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5459728031419218, 'learning_rate': 9.476831316104167e-06, 'epoch': 17.6})   +03/15 [12:31:16] INFO  | >> Step 7300, Loss: {'action_dit_loss': 0.011605553328990936, 'mse_score': 0.0023089315742254256, 'data_time': ]8;id=632556;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=941423;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.003015827853232622, 'model_time': 2.515078474767506, 'learning_rate': 9.453777180309831e-06, 'epoch': 17.72})   +03/15 [12:33:25] INFO  | >> Step 7350, Loss: {'action_dit_loss': 0.017023947089910507, 'data_time': 0.00017621880397200584, 'model_time': ]8;id=457007;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=509232;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.526497332844883, 'learning_rate': 9.430256569911963e-06, 'epoch': 17.84})   +03/15 [12:35:32] INFO  | >> Step 7400, Loss: {'action_dit_loss': 0.0836988314986229, 'mse_score': 0.0022534100338816644, 'data_time': ]8;id=712131;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=836912;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001355574931949377, 'model_time': 2.511962963268161, 'learning_rate': 9.406272064210546e-06, 'epoch': 17.96})   +03/15 [12:37:38] INFO  | >> Step 7450, Loss: {'action_dit_loss': 0.03366437926888466, 'data_time': 0.0023318701423704624, 'model_time': ]8;id=490094;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=964209;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5566607578657568, 'learning_rate': 9.38182629337691e-06, 'epoch': 18.08})   +03/15 [12:39:45] INFO  | >> Step 7500, Loss: {'action_dit_loss': 0.038898926228284836, 'mse_score': 0.002863340638577938, 'data_time': ]8;id=950983;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=413160;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015368168242275715, 'model_time': 2.530453145969659, 'learning_rate': 9.356921938165308e-06, 'epoch': 18.2})   +03/15 [12:41:52] INFO  | >> Step 7550, Loss: {'action_dit_loss': 0.018733764067292213, 'data_time': 0.0016712751239538193, 'model_time': ]8;id=88025;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=673971;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5143552026711404, 'learning_rate': 9.33156172961893e-06, 'epoch': 18.33})   +03/15 [12:44:00] INFO  | >> Step 7600, Loss: {'action_dit_loss': 0.024367239326238632, 'mse_score': 0.001981434039771557, 'data_time': ]8;id=190556;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=52727;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001747608184814453, 'model_time': 2.7021235250867903, 'learning_rate': 9.305748448770433e-06, 'epoch': 18.45})   +03/15 [12:46:07] INFO  | >> Step 7650, Loss: {'action_dit_loss': 0.03701992705464363, 'data_time': 0.0014641070738434792, 'model_time': ]8;id=476786;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=342722;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.526764434762299, 'learning_rate': 9.279484926336948e-06, 'epoch': 18.57})   +03/15 [12:48:14] INFO  | >> Step 7700, Loss: {'action_dit_loss': 0.01867302507162094, 'mse_score': 0.003716382756829262, 'data_time': ]8;id=291773;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=788539;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002307157963514328, 'model_time': 2.5496981381438673, 'learning_rate': 9.252774042409674e-06, 'epoch': 18.69})   +03/15 [12:50:21] INFO  | >> Step 7750, Loss: {'action_dit_loss': 0.05050010234117508, 'data_time': 0.00013837963342666626, 'model_time': ]8;id=875467;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=85884;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.529650520067662, 'learning_rate': 9.225618726138036e-06, 'epoch': 18.81})   +03/15 [12:52:29] INFO  | >> Step 7800, Loss: {'action_dit_loss': 0.03853241726756096, 'mse_score': 0.0024109724909067154, 'data_time': ]8;id=54615;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=998160;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001627202145755291, 'model_time': 2.5231018937192857, 'learning_rate': 9.198021955408478e-06, 'epoch': 18.93})   +03/15 [12:54:35] INFO  | >> Step 7850, Loss: {'action_dit_loss': 0.046277012676000595, 'data_time': 0.004008031915873289, 'model_time': ]8;id=819182;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=683414;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5264561427757144, 'learning_rate': 9.169986756517897e-06, 'epoch': 19.05})   +03/15 [12:56:44] INFO  | >> Step 7900, Loss: {'action_dit_loss': 0.02345840446650982, 'mse_score': 0.0016003431752324103, 'data_time': ]8;id=259295;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=209044;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017368001863360405, 'model_time': 2.542749141342938, 'learning_rate': 9.14151620384178e-06, 'epoch': 19.17})   +03/15 [12:58:50] INFO  | >> Step 7950, Loss: {'action_dit_loss': 0.018376139923930168, 'data_time': 0.00016264989972114563, 'model_time': ]8;id=250132;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=132352;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5131732020527124, 'learning_rate': 9.11261341949706e-06, 'epoch': 19.3})   +03/15 [13:00:57] INFO  | >> Step 8000, Loss: {'action_dit_loss': 0.047058042138814926, 'mse_score': 0.0014370819553732872, 'data_time': ]8;id=993795;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=228552;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001749400980770588, 'model_time': 2.5417520101182163, 'learning_rate': 9.083281572999748e-06, 'epoch': 19.42})   +03/15 [13:03:04] INFO  | >> Step 8050, Loss: {'action_dit_loss': 0.09446268528699875, 'data_time': 0.00017020013183355331, 'model_time': ]8;id=386814;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=175939;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.56414448376745, 'learning_rate': 9.05352388091735e-06, 'epoch': 19.54})   +03/15 [13:05:11] INFO  | >> Step 8100, Loss: {'action_dit_loss': 0.06613758951425552, 'mse_score': 0.0018656682223081589, 'data_time': ]8;id=120096;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=815573;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.003843474667519331, 'model_time': 2.538195736706257, 'learning_rate': 9.023343606516143e-06, 'epoch': 19.66})   +03/15 [13:07:18] INFO  | >> Step 8150, Loss: {'action_dit_loss': 0.023452792316675186, 'data_time': 0.002472215797752142, 'model_time': ]8;id=606794;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=26925;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.7993492130190134, 'learning_rate': 8.992744059403315e-06, 'epoch': 19.78})   +03/15 [13:09:27] INFO  | >> Step 8200, Loss: {'action_dit_loss': 0.026503199711441994, 'mse_score': 0.002458912134170532, 'data_time': ]8;id=952134;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=393537;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001557399518787861, 'model_time': 2.535264703910798, 'learning_rate': 8.961728595164032e-06, 'epoch': 19.9})   +03/15 [13:11:33] INFO  | >> Step 8250, Loss: {'action_dit_loss': 0.029183922335505486, 'data_time': 0.004113142378628254, 'model_time': ]8;id=79688;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=620859;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.525871174875647, 'learning_rate': 8.930300614993462e-06, 'epoch': 20.02})   +03/15 [13:13:41] INFO  | >> Step 8300, Loss: {'action_dit_loss': 0.011667969636619091, 'mse_score': 0.0015954773873090744, 'data_time': ]8;id=106851;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=731059;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017292890697717667, 'model_time': 2.5053953430615366, 'learning_rate': 8.898463565323789e-06, 'epoch': 20.15})   +03/15 [13:15:47] INFO  | >> Step 8350, Loss: {'action_dit_loss': 0.06547775119543076, 'data_time': 0.0001714928075671196, 'model_time': ]8;id=629577;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=844657;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5145375430583954, 'learning_rate': 8.866220937446284e-06, 'epoch': 20.27})   +03/15 [13:17:54] INFO  | >> Step 8400, Loss: {'action_dit_loss': 0.013768322765827179, 'mse_score': 0.0019094491377472877, 'data_time': ]8;id=43064;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=364069;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017013400793075562, 'model_time': 2.507355240639299, 'learning_rate': 8.833576267128432e-06, 'epoch': 20.39})   +03/15 [13:20:02] INFO  | >> Step 8450, Loss: {'action_dit_loss': 0.02417301945388317, 'data_time': 0.0016864398494362831, 'model_time': ]8;id=72306;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=530538;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5243325578048825, 'learning_rate': 8.800533134226206e-06, 'epoch': 20.51})   +03/15 [13:22:09] INFO  | >> Step 8500, Loss: {'action_dit_loss': 0.06216317415237427, 'mse_score': 0.0037151511758565904, 'data_time': ]8;id=440463;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=862276;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002470589242875576, 'model_time': 2.5312915588729084, 'learning_rate': 8.767095162291493e-06, 'epoch': 20.63})   +03/15 [13:24:15] INFO  | >> Step 8550, Loss: {'action_dit_loss': 0.019516825675964355, 'data_time': 0.00015592621639370918, 'model_time': ]8;id=666459;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=934884;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5161893889307976, 'learning_rate': 8.733266018174727e-06, 'epoch': 20.75})   +03/15 [13:26:24] INFO  | >> Step 8600, Loss: {'action_dit_loss': 0.09761092811822891, 'mse_score': 0.0046741548925638195, 'data_time': ]8;id=456641;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=184692;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00014774687588214874, 'model_time': 2.5079143061302602, 'learning_rate': 8.699049411622776e-06, 'epoch': 20.87})   +03/15 [13:28:30] INFO  | >> Step 8650, Loss: {'action_dit_loss': 0.015084645710885525, 'data_time': 0.002447111066430807, 'model_time': ]8;id=645830;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=847458;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5355838984251022, 'learning_rate': 8.664449094872139e-06, 'epoch': 21.0})   +03/15 [13:30:37] INFO  | >> Step 8700, Loss: {'action_dit_loss': 0.03364024683833122, 'mse_score': 0.0017977483570575715, 'data_time': ]8;id=487455;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=456732;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015854835510253906, 'model_time': 2.5129338591359556, 'learning_rate': 8.629468862237456e-06, 'epoch': 21.12})   +03/15 [13:32:44] INFO  | >> Step 8750, Loss: {'action_dit_loss': 0.017317911610007286, 'data_time': 0.0014286618679761887, 'model_time': ]8;id=337960;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=893085;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5439302888698876, 'learning_rate': 8.594112549695429e-06, 'epoch': 21.24})   +03/15 [13:34:51] INFO  | >> Step 8800, Loss: {'action_dit_loss': 0.011700859293341637, 'mse_score': 0.0019765397533774375, 'data_time': ]8;id=292477;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=924457;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016751792281866074, 'model_time': 2.5175821068696678, 'learning_rate': 8.558384034464155e-06, 'epoch': 21.36})   +03/15 [13:36:58] INFO  | >> Step 8850, Loss: {'action_dit_loss': 0.030816365033388138, 'data_time': 0.00018161814659833908, 'model_time': ]8;id=597530;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=639979;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5324975359253585, 'learning_rate': 8.522287234577954e-06, 'epoch': 21.48})   +03/15 [13:39:05] INFO  | >> Step 8900, Loss: {'action_dit_loss': 0.010701115243136883, 'mse_score': 0.0015527422539889813, 'data_time': ]8;id=518317;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=892288;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0030397693626582623, 'model_time': 2.533699966967106, 'learning_rate': 8.485826108457706e-06, 'epoch': 21.6})   +03/15 [13:41:12] INFO  | >> Step 8950, Loss: {'action_dit_loss': 0.044742025434970856, 'data_time': 0.00015523796901106834, 'model_time': ]8;id=372060;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=836587;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5415683719329536, 'learning_rate': 8.449004654476772e-06, 'epoch': 21.72})   +03/15 [13:43:20] INFO  | >> Step 9000, Loss: {'action_dit_loss': 0.015621802769601345, 'mse_score': 0.0011842258274555207, 'data_time': ]8;id=625113;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=735244;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015942798927426338, 'model_time': 2.5305376816540956, 'learning_rate': 8.411826910522521e-06, 'epoch': 21.84})   +03/15 [13:45:27] INFO  | >> Step 9050, Loss: {'action_dit_loss': 0.020860036835074425, 'data_time': 0.0029992139898240566, 'model_time': ]8;id=541722;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=994115;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5178279927931726, 'learning_rate': 8.374296953553528e-06, 'epoch': 21.97})   +03/15 [13:47:34] INFO  | >> Step 9100, Loss: {'action_dit_loss': 0.014484134502708912, 'mse_score': 0.000976772326976061, 'data_time': ]8;id=426170;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=512311;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016745878383517265, 'model_time': 2.537125817965716, 'learning_rate': 8.336418899152507e-06, 'epoch': 22.09})   +03/15 [13:49:40] INFO  | >> Step 9150, Loss: {'action_dit_loss': 0.038735873997211456, 'data_time': 0.00015651201829314232, 'model_time': ]8;id=499219;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=677278;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.511995510663837, 'learning_rate': 8.298196901074964e-06, 'epoch': 22.21})   +03/15 [13:51:47] INFO  | >> Step 9200, Loss: {'action_dit_loss': 0.02459838055074215, 'mse_score': 0.0010830331593751907, 'data_time': ]8;id=18080;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=97573;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017924094572663307, 'model_time': 2.537955607753247, 'learning_rate': 8.259635150793712e-06, 'epoch': 22.33})   +03/15 [13:53:54] INFO  | >> Step 9250, Loss: {'action_dit_loss': 0.03904825448989868, 'data_time': 0.00017149001359939575, 'model_time': ]8;id=255123;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=321080;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.508049003779888, 'learning_rate': 8.22073787703922e-06, 'epoch': 22.45})   +03/15 [13:56:02] INFO  | >> Step 9300, Loss: {'action_dit_loss': 0.07007447630167007, 'mse_score': 0.002402869611978531, 'data_time': ]8;id=580358;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=556711;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0035578268580138683, 'model_time': 2.5043072402477264, 'learning_rate': 8.18150934533589e-06, 'epoch': 22.57})   +03/15 [13:58:09] INFO  | >> Step 9350, Loss: {'action_dit_loss': 0.02091771364212036, 'data_time': 0.0001752600073814392, 'model_time': ]8;id=346859;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=368900;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.54468339914456, 'learning_rate': 8.141953857534288e-06, 'epoch': 22.69})   +03/15 [14:00:17] INFO  | >> Step 9400, Loss: {'action_dit_loss': 0.0436578206717968, 'mse_score': 0.0011946487240493298, 'data_time': ]8;id=263615;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=241736;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001696101389825344, 'model_time': 2.544292998034507, 'learning_rate': 8.102075751339402e-06, 'epoch': 22.82})   +03/15 [14:02:24] INFO  | >> Step 9450, Loss: {'action_dit_loss': 0.007666380610316992, 'data_time': 0.0032719532027840614, 'model_time': ]8;id=125362;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=778967;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.534781435970217, 'learning_rate': 8.061879399834956e-06, 'epoch': 22.94})   +03/15 [14:04:31] INFO  | >> Step 9500, Loss: {'action_dit_loss': 0.05724421143531799, 'mse_score': 0.0019441615790128709, 'data_time': ]8;id=194143;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=200845;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017363997176289558, 'model_time': 2.552795419935137, 'learning_rate': 8.021369211003872e-06, 'epoch': 23.06})   +03/15 [14:06:38] INFO  | >> Step 9550, Loss: {'action_dit_loss': 0.017398804426193237, 'data_time': 0.001361784990876913, 'model_time': ]8;id=759782;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=618226;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5230837147682905, 'learning_rate': 7.980549627244858e-06, 'epoch': 23.18})   +03/15 [14:08:45] INFO  | >> Step 9600, Loss: {'action_dit_loss': 0.013638192787766457, 'mse_score': 0.002215721271932125, 'data_time': ]8;id=105409;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=873074;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001646401360630989, 'model_time': 2.5276516103185713, 'learning_rate': 7.939425124885275e-06, 'epoch': 23.3})   +03/15 [14:10:52] INFO  | >> Step 9650, Loss: {'action_dit_loss': 0.022699834778904915, 'data_time': 0.00015687895938754082, 'model_time': ]8;id=188158;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=316950;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5198058038949966, 'learning_rate': 7.898000213690227e-06, 'epoch': 23.42})   +03/15 [14:12:59] INFO  | >> Step 9700, Loss: {'action_dit_loss': 0.00647653266787529, 'mse_score': 0.0014195895753800869, 'data_time': ]8;id=287636;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=47726;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004106531850993633, 'model_time': 2.5067440760321915, 'learning_rate': 7.856279436368053e-06, 'epoch': 23.54})   +03/15 [14:15:05] INFO  | >> Step 9750, Loss: {'action_dit_loss': 0.012869752012193203, 'data_time': 0.00018362980335950851, 'model_time': ]8;id=989949;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=132413;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5218155309557915, 'learning_rate': 7.814267368072132e-06, 'epoch': 23.67})   +03/15 [14:17:13] INFO  | >> Step 9800, Loss: {'action_dit_loss': 0.025401616469025612, 'mse_score': 0.0011324086226522923, 'data_time': ]8;id=107570;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=915112;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001547493040561676, 'model_time': 2.528483809903264, 'learning_rate': 7.771968615899184e-06, 'epoch': 23.79})   +03/15 [14:19:21] INFO  | >> Step 9850, Loss: {'action_dit_loss': 0.0368492491543293, 'data_time': 0.0016016019508242607, 'model_time': ]8;id=501968;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=461865;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5179246673360467, 'learning_rate': 7.729387818384054e-06, 'epoch': 23.91})   +03/15 [14:21:28] INFO  | >> Step 9900, Loss: {'action_dit_loss': 0.016891637817025185, 'mse_score': 0.0020652193576097488, 'data_time': ]8;id=986776;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=903582;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001556086353957653, 'model_time': 2.550581792835146, 'learning_rate': 7.686529644991026e-06, 'epoch': 24.03})   +03/15 [14:23:34] INFO  | >> Step 9950, Loss: {'action_dit_loss': 0.09663347154855728, 'data_time': 0.00017644325271248817, 'model_time': ]8;id=420172;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=515633;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5085345190018415, 'learning_rate': 7.643398795601783e-06, 'epoch': 24.15})   +03/15 [14:25:42] INFO  | >> Step 10000, Loss: {'action_dit_loss': 0.033615656197071075, 'mse_score': 0.0013367464765906335, 'data_time': ]8;id=56213;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=159097;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0024528237991034985, 'model_time': 2.5374284610152245, 'learning_rate': 7.600000000000001e-06, 'epoch': 24.27})   +✅ Checkpoint saved at ./results/Checkpoints/fastumi_pickandplace_qwenPI/checkpoints/steps_10000 +03/15 [14:25:50] INFO  | >> 📊 Saving accessed configuration... ]8;id=318594;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=89318;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#240\240]8;;\ +  INFO  | >> ✅ Configuration files saved ]8;id=436390;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=635770;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#243\243]8;;\ +03/15 [14:27:56] INFO  | >> Step 10050, Loss: {'action_dit_loss': 0.022575369104743004, 'data_time': 0.00016354210674762726, 'model_time': ]8;id=813373;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=547957;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5462885042652488, 'learning_rate': 7.556338017352677e-06, 'epoch': 24.39})   +03/15 [14:30:04] INFO  | >> Step 10100, Loss: {'action_dit_loss': 0.008738790638744831, 'mse_score': 0.0023369822651147843, 'data_time': ]8;id=311797;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=901950;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.003438096959143877, 'model_time': 2.774251312017441, 'learning_rate': 7.512417635688234e-06, 'epoch': 24.51})   +03/15 [14:32:11] INFO  | >> Step 10150, Loss: {'action_dit_loss': 0.024879761040210724, 'data_time': 0.0024158242158591747, 'model_time': ]8;id=651194;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=63156;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5024778321385384, 'learning_rate': 7.46824367137146e-06, 'epoch': 24.64})   +03/15 [14:34:19] INFO  | >> Step 10200, Loss: {'action_dit_loss': 0.041553713381290436, 'mse_score': 0.0016192914918065071, 'data_time': ]8;id=799787;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=217881;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  1.0855432599782944, 'model_time': 2.560619103256613, 'learning_rate': 7.423820968575337e-06, 'epoch': 24.76})   +03/15 [14:36:26] INFO  | >> Step 10250, Loss: {'action_dit_loss': 0.029933560639619827, 'data_time': 0.001626133918762207, 'model_time': ]8;id=85131;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=164686;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.52105525881052, 'learning_rate': 7.3791543987498245e-06, 'epoch': 24.88})   +03/15 [14:38:33] INFO  | >> Step 10300, Loss: {'action_dit_loss': 0.012461570091545582, 'mse_score': 0.0011406936682760715, 'data_time': ]8;id=164136;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=2805;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016850000247359276, 'model_time': 2.547414109110832, 'learning_rate': 7.3342488600876525e-06, 'epoch': 25.0})   +03/15 [14:40:39] INFO  | >> Step 10350, Loss: {'action_dit_loss': 0.02294733002781868, 'data_time': 0.003321512136608362, 'model_time': ]8;id=492738;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=305407;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.538221552968025, 'learning_rate': 7.289109276987173e-06, 'epoch': 25.12})   +03/15 [14:42:47] INFO  | >> Step 10400, Loss: {'action_dit_loss': 0.01034932304173708, 'mse_score': 0.0021746084094047546, 'data_time': ]8;id=296451;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=737135;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016910023987293243, 'model_time': 2.5041426876559854, 'learning_rate': 7.24374059951235e-06, 'epoch': 25.24})   +03/15 [14:44:54] INFO  | >> Step 10450, Loss: {'action_dit_loss': 0.020821331068873405, 'data_time': 0.00016400963068008423, 'model_time': ]8;id=244762;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=968790;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5119935269467533, 'learning_rate': 7.1981478028499265e-06, 'epoch': 25.36})   +03/15 [14:47:01] INFO  | >> Step 10500, Loss: {'action_dit_loss': 0.008551242761313915, 'mse_score': 0.0011107315309345721, 'data_time': ]8;id=618439;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=693300;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0026551769115030766, 'model_time': 2.5296559650450945, 'learning_rate': 7.1523358867638425e-06, 'epoch': 25.49})   +03/15 [14:49:08] INFO  | >> Step 10550, Loss: {'action_dit_loss': 0.016779644414782524, 'data_time': 0.00016313977539539337, 'model_time': ]8;id=120351;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=571015;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.516519149299711, 'learning_rate': 7.106309875046947e-06, 'epoch': 25.61})   +03/15 [14:51:15] INFO  | >> Step 10600, Loss: {'action_dit_loss': 0.05006298050284386, 'mse_score': 0.0037285521626472472, 'data_time': ]8;id=278535;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=866785;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0014337580651044846, 'model_time': 2.509338327217847, 'learning_rate': 7.060074814970095e-06, 'epoch': 25.73})   +03/15 [14:53:23] INFO  | >> Step 10650, Loss: {'action_dit_loss': 0.011821649968624115, 'data_time': 0.004715892020612955, 'model_time': ]8;id=831131;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=322554;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5203936281614006, 'learning_rate': 7.013635776728643e-06, 'epoch': 25.85})   +03/15 [14:55:30] INFO  | >> Step 10700, Loss: {'action_dit_loss': 0.013303227722644806, 'mse_score': 0.0018273545429110527, 'data_time': ]8;id=965940;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=302630;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015474902465939522, 'model_time': 2.522749848663807, 'learning_rate': 6.966997852886455e-06, 'epoch': 25.97})   +03/15 [14:57:36] INFO  | >> Step 10750, Loss: {'action_dit_loss': 0.029438737779855728, 'data_time': 0.00017943885177373886, 'model_time': ]8;id=318862;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=733723;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5040061962790787, 'learning_rate': 6.92016615781744e-06, 'epoch': 26.09})   +03/15 [14:59:43] INFO  | >> Step 10800, Loss: {'action_dit_loss': 0.023620106279850006, 'mse_score': 0.0025977689772844316, 'data_time': ]8;id=566211;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=517781;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015479270368814468, 'model_time': 2.527291870675981, 'learning_rate': 6.873145827144714e-06, 'epoch': 26.21})   +03/15 [15:01:50] INFO  | >> Step 10850, Loss: {'action_dit_loss': 0.02975093200802803, 'data_time': 0.0023730443790555, 'model_time': ]8;id=932627;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=453014;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.506992624141276, 'learning_rate': 6.8259420171774005e-06, 'epoch': 26.33})   +03/15 [15:03:56] INFO  | >> Step 10900, Loss: {'action_dit_loss': 0.02539096213877201, 'mse_score': 0.001187324058264494, 'data_time': ]8;id=27121;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=95763;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016660289838910103, 'model_time': 2.5495351902209222, 'learning_rate': 6.7785599043452e-06, 'epoch': 26.46})   +03/15 [15:06:04] INFO  | >> Step 10950, Loss: {'action_dit_loss': 0.02033301442861557, 'data_time': 0.001630687154829502, 'model_time': ]8;id=603139;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=615643;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5110427322797477, 'learning_rate': 6.731004684630736e-06, 'epoch': 26.58})   +03/15 [15:08:11] INFO  | >> Step 11000, Loss: {'action_dit_loss': 0.01762121357023716, 'mse_score': 0.0013645419850945473, 'data_time': ]8;id=861322;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=282602;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017454475164413452, 'model_time': 2.5318822022527456, 'learning_rate': 6.683281572999748e-06, 'epoch': 26.7})   +03/15 [15:10:19] INFO  | >> Step 11050, Loss: {'action_dit_loss': 0.012914923019707203, 'data_time': 0.0026316638104617596, 'model_time': ]8;id=183700;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=493355;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5339459436945617, 'learning_rate': 6.635395802829216e-06, 'epoch': 26.82})   +03/15 [15:12:26] INFO  | >> Step 11100, Loss: {'action_dit_loss': 0.013889849185943604, 'mse_score': 0.001833157241344452, 'data_time': ]8;id=291668;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=190305;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0014737872406840324, 'model_time': 2.539852349087596, 'learning_rate': 6.587352625333464e-06, 'epoch': 26.94})   +03/15 [15:14:33] INFO  | >> Step 11150, Loss: {'action_dit_loss': 0.03112267516553402, 'data_time': 0.0015293378382921219, 'model_time': ]8;id=515611;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=95674;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5045350138098, 'learning_rate': 6.539157308988302e-06, 'epoch': 27.06})   +03/15 [15:16:41] INFO  | >> Step 11200, Loss: {'action_dit_loss': 0.037771761417388916, 'mse_score': 0.0011478805914521216, 'data_time': ]8;id=336653;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=702685;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016138702630996704, 'model_time': 2.5584336617030203, 'learning_rate': 6.490815138953276e-06, 'epoch': 27.18})   +03/15 [15:18:47] INFO  | >> Step 11250, Loss: {'action_dit_loss': 0.0183711014688015, 'data_time': 0.0012465179897844791, 'model_time': ]8;id=431641;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=727369;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.514838010072708, 'learning_rate': 6.4423314164921005e-06, 'epoch': 27.31})   +03/15 [15:20:54] INFO  | >> Step 11300, Loss: {'action_dit_loss': 0.007697753608226776, 'mse_score': 0.0013835165649652482, 'data_time': ]8;id=419904;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=853085;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.011226091999560595, 'model_time': 2.5170632638037205, 'learning_rate': 6.393711458391305e-06, 'epoch': 27.43})   +03/15 [15:23:01] INFO  | >> Step 11350, Loss: {'action_dit_loss': 0.014235523529350758, 'data_time': 0.00014947820454835892, 'model_time': ]8;id=92337;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=329795;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5321614709682763, 'learning_rate': 6.344960596377189e-06, 'epoch': 27.55})   +03/15 [15:25:08] INFO  | >> Step 11400, Loss: {'action_dit_loss': 0.010943290777504444, 'mse_score': 0.0015662075951695443, 'data_time': ]8;id=423808;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=906924;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002342723309993744, 'model_time': 2.5223745559342206, 'learning_rate': 6.296084176531149e-06, 'epoch': 27.67})   +03/15 [15:27:16] INFO  | >> Step 11450, Loss: {'action_dit_loss': 0.049800194799900055, 'data_time': 0.002537062391638756, 'model_time': ]8;id=911597;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=568950;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.523346171248704, 'learning_rate': 6.247087558703405e-06, 'epoch': 27.79})   +03/15 [15:29:24] INFO  | >> Step 11500, Loss: {'action_dit_loss': 0.027870429679751396, 'mse_score': 0.0011465780436992646, 'data_time': ]8;id=543623;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=379353;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001653078943490982, 'model_time': 2.5415870882570744, 'learning_rate': 6.197976115925245e-06, 'epoch': 27.91})   +03/15 [15:31:31] INFO  | >> Step 11550, Loss: {'action_dit_loss': 0.020756779238581657, 'data_time': 0.002406734973192215, 'model_time': ]8;id=463513;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=796907;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.545897088944912, 'learning_rate': 6.148755233819807e-06, 'epoch': 28.03})   +03/15 [15:33:38] INFO  | >> Step 11600, Loss: {'action_dit_loss': 0.01378451194614172, 'mse_score': 0.0010172595269978047, 'data_time': ]8;id=137377;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=972735;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001666671596467495, 'model_time': 2.5089514767751098, 'learning_rate': 6.0994303100114795e-06, 'epoch': 28.16})   +03/15 [15:35:45] INFO  | >> Step 11650, Loss: {'action_dit_loss': 0.01653790846467018, 'data_time': 0.00017151469364762306, 'model_time': ]8;id=508257;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=127324;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5433853324502707, 'learning_rate': 6.050006753534001e-06, 'epoch': 28.28})   +03/15 [15:37:52] INFO  | >> Step 11700, Loss: {'action_dit_loss': 0.009764840826392174, 'mse_score': 0.002325694262981415, 'data_time': ]8;id=250921;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=744261;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015536239370703697, 'model_time': 2.5432810140773654, 'learning_rate': 6.0004899842372925e-06, 'epoch': 28.4})   +03/15 [15:39:59] INFO  | >> Step 11750, Loss: {'action_dit_loss': 0.03144737705588341, 'data_time': 0.002782444003969431, 'model_time': ]8;id=579077;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=427809;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5170231168158352, 'learning_rate': 5.950885432193109e-06, 'epoch': 28.52})   +03/15 [15:42:06] INFO  | >> Step 11800, Loss: {'action_dit_loss': 0.0187755785882473, 'mse_score': 0.0012722622603178024, 'data_time': ]8;id=118972;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=483863;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0016368292272090912, 'model_time': 2.5118043636903167, 'learning_rate': 5.901198537099578e-06, 'epoch': 28.64})   +03/15 [15:44:14] INFO  | >> Step 11850, Loss: {'action_dit_loss': 0.030527764931321144, 'data_time': 0.0039011286571621895, 'model_time': ]8;id=161450;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=522584;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.551203416660428, 'learning_rate': 5.851434747684662e-06, 'epoch': 28.76})   +03/15 [15:46:21] INFO  | >> Step 11900, Loss: {'action_dit_loss': 0.009014353156089783, 'mse_score': 0.001351218018680811, 'data_time': ]8;id=739677;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=286706;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0002720709890127182, 'model_time': 2.505409063305706, 'learning_rate': 5.801599521108661e-06, 'epoch': 28.88})   +03/15 [15:48:28] INFO  | >> Step 11950, Loss: {'action_dit_loss': 0.012541431002318859, 'data_time': 0.00015668105334043503, 'model_time': ]8;id=255570;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=478975;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.543570522684604, 'learning_rate': 5.751698322365761e-06, 'epoch': 29.0})   +03/15 [15:50:35] INFO  | >> Step 12000, Loss: {'action_dit_loss': 0.015011143870651722, 'mse_score': 0.0013332199305295944, 'data_time': ]8;id=966648;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=628492;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016954075545072556, 'model_time': 2.5483785890974104, 'learning_rate': 5.701736623684737e-06, 'epoch': 29.13})   +03/15 [15:52:43] INFO  | >> Step 12050, Loss: {'action_dit_loss': 0.007791229523718357, 'data_time': 0.0003788592293858528, 'model_time': ]8;id=905997;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=73201;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.510099488310516, 'learning_rate': 5.65171990392887e-06, 'epoch': 29.25})   +03/15 [15:54:49] INFO  | >> Step 12100, Loss: {'action_dit_loss': 0.0076170265674591064, 'mse_score': 0.001803208515048027, 'data_time': ]8;id=435098;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=356385;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.003763824235647917, 'model_time': 2.5456335740163922, 'learning_rate': 5.6016536479951165e-06, 'epoch': 29.37})   +03/15 [15:56:56] INFO  | >> Step 12150, Loss: {'action_dit_loss': 0.006412095855921507, 'data_time': 0.00019509997218847275, 'model_time': ]8;id=860408;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=2689;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.506670563016087, 'learning_rate': 5.551543346212632e-06, 'epoch': 29.49})   +03/15 [15:59:03] INFO  | >> Step 12200, Loss: {'action_dit_loss': 0.020536327734589577, 'mse_score': 0.0008865566924214363, 'data_time': ]8;id=615448;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=608093;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001902598887681961, 'model_time': 2.508153055328876, 'learning_rate': 5.501394493740705e-06, 'epoch': 29.61})   +03/15 [16:01:11] INFO  | >> Step 12250, Loss: {'action_dit_loss': 0.011217208579182625, 'data_time': 0.004036115948110819, 'model_time': ]8;id=468238;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=564742;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5206258818507195, 'learning_rate': 5.451212589966132e-06, 'epoch': 29.73})   +03/15 [16:03:18] INFO  | >> Step 12300, Loss: {'action_dit_loss': 0.011178155429661274, 'mse_score': 0.0015996504575014115, 'data_time': ]8;id=799929;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=569827;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015509040094912052, 'model_time': 2.507465303875506, 'learning_rate': 5.401003137900158e-06, 'epoch': 29.85})   +03/15 [16:05:25] INFO  | >> Step 12350, Loss: {'action_dit_loss': 0.014223091304302216, 'data_time': 0.00016728462651371956, 'model_time': ]8;id=911843;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=197816;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.543656115885824, 'learning_rate': 5.350771643575015e-06, 'epoch': 29.98})   +03/15 [16:07:32] INFO  | >> Step 12400, Loss: {'action_dit_loss': 0.010724392719566822, 'mse_score': 0.0013177287764847279, 'data_time': ]8;id=244907;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=897594;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017075426876544952, 'model_time': 2.5152138718403876, 'learning_rate': 5.300523615440114e-06, 'epoch': 30.1})   +03/15 [16:09:39] INFO  | >> Step 12450, Loss: {'action_dit_loss': 0.02555244043469429, 'data_time': 0.00015624286606907845, 'model_time': ]8;id=780879;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=495951;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.766397678758949, 'learning_rate': 5.250264563757981e-06, 'epoch': 30.22})   +03/15 [16:11:46] INFO  | >> Step 12500, Loss: {'action_dit_loss': 0.03871709853410721, 'mse_score': 0.0026644615456461905, 'data_time': ]8;id=404832;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=695928;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.003927871119230986, 'model_time': 2.5474798469804227, 'learning_rate': 5.2e-06, 'epoch': 30.34})   +03/15 [16:13:52] INFO  | >> Step 12550, Loss: {'action_dit_loss': 0.01355693954974413, 'data_time': 0.00015614088624715805, 'model_time': ]8;id=519389;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=38817;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5292847706004977, 'learning_rate': 5.149735436242021e-06, 'epoch': 30.46})   +03/15 [16:16:00] INFO  | >> Step 12600, Loss: {'action_dit_loss': 0.0067426967434585094, 'mse_score': 0.0009713754057884217, 'data_time': ]8;id=911946;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=105239;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015892088413238525, 'model_time': 2.5361740249209106, 'learning_rate': 5.099476384559887e-06, 'epoch': 30.58})   +03/15 [16:18:08] INFO  | >> Step 12650, Loss: {'action_dit_loss': 0.011775736697018147, 'data_time': 0.004100433085113764, 'model_time': ]8;id=551499;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=954667;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5078332307748497, 'learning_rate': 5.049228356424984e-06, 'epoch': 30.7})   +03/15 [16:20:15] INFO  | >> Step 12700, Loss: {'action_dit_loss': 0.010679231956601143, 'mse_score': 0.0012129543349146843, 'data_time': ]8;id=429893;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=913239;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001666816882789135, 'model_time': 2.521894872188568, 'learning_rate': 4.998996862099842e-06, 'epoch': 30.83})   +03/15 [16:22:22] INFO  | >> Step 12750, Loss: {'action_dit_loss': 0.03432011231780052, 'data_time': 0.00016778986901044846, 'model_time': ]8;id=819488;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=277896;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5193028720095754, 'learning_rate': 4.948787410033869e-06, 'epoch': 30.95})   +03/15 [16:24:29] INFO  | >> Step 12800, Loss: {'action_dit_loss': 0.00779710803180933, 'mse_score': 0.0019147815182805061, 'data_time': ]8;id=681403;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=84216;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016515003517270088, 'model_time': 2.543540464248508, 'learning_rate': 4.898605506259296e-06, 'epoch': 31.07})   +03/15 [16:26:35] INFO  | >> Step 12850, Loss: {'action_dit_loss': 0.02007286250591278, 'data_time': 0.0015858621336519718, 'model_time': ]8;id=901085;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=559543;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.510670145973563, 'learning_rate': 4.848456653787369e-06, 'epoch': 31.19})   +03/15 [16:28:43] INFO  | >> Step 12900, Loss: {'action_dit_loss': 0.006473332643508911, 'mse_score': 0.0017308153212070466, 'data_time': ]8;id=930546;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=796290;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.003801172599196434, 'model_time': 2.556882280856371, 'learning_rate': 4.798346352004886e-06, 'epoch': 31.31})   +03/15 [16:30:49] INFO  | >> Step 12950, Loss: {'action_dit_loss': 0.02034144662320614, 'data_time': 0.000158610288053751, 'model_time': ]8;id=647453;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=71744;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5125591228716075, 'learning_rate': 4.748280096071131e-06, 'epoch': 31.43})   +03/15 [16:32:56] INFO  | >> Step 13000, Loss: {'action_dit_loss': 0.014538350515067577, 'mse_score': 0.002575038932263851, 'data_time': ]8;id=301338;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=238536;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001781904138624668, 'model_time': 2.5077771302312613, 'learning_rate': 4.6982633763152645e-06, 'epoch': 31.55})   +03/15 [16:35:03] INFO  | >> Step 13050, Loss: {'action_dit_loss': 0.015243974514305592, 'data_time': 0.00350766908377409, 'model_time': ]8;id=797305;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=664532;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.499837063252926, 'learning_rate': 4.648301677634241e-06, 'epoch': 31.67})   +03/15 [16:37:11] INFO  | >> Step 13100, Loss: {'action_dit_loss': 0.026858506724238396, 'mse_score': 0.0017826491966843605, 'data_time': ]8;id=174446;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=727712;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001768600195646286, 'model_time': 2.5418405886739492, 'learning_rate': 4.598400478891339e-06, 'epoch': 31.8})   +03/15 [16:39:18] INFO  | >> Step 13150, Loss: {'action_dit_loss': 0.01507542748004198, 'data_time': 0.0038940799422562122, 'model_time': ]8;id=340131;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=834816;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5118146329186857, 'learning_rate': 4.54856525231534e-06, 'epoch': 31.92})   +03/15 [16:41:25] INFO  | >> Step 13200, Loss: {'action_dit_loss': 0.01717389188706875, 'mse_score': 0.0016734382137656213, 'data_time': ]8;id=451607;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=152640;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016873283311724663, 'model_time': 2.504607822280377, 'learning_rate': 4.4988014629004244e-06, 'epoch': 32.04})   +03/15 [16:43:31] INFO  | >> Step 13250, Loss: {'action_dit_loss': 0.06136597320437431, 'data_time': 0.00016579404473304749, 'model_time': ]8;id=714919;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=831016;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5496800770051777, 'learning_rate': 4.449114567806891e-06, 'epoch': 32.16})   +03/15 [16:45:38] INFO  | >> Step 13300, Loss: {'action_dit_loss': 0.015681428834795952, 'mse_score': 0.001376395206898451, 'data_time': ]8;id=639098;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=913141;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015610349364578724, 'model_time': 2.540180301759392, 'learning_rate': 4.3995100157627095e-06, 'epoch': 32.28})   +03/15 [16:47:44] INFO  | >> Step 13350, Loss: {'action_dit_loss': 0.03163161501288414, 'data_time': 0.0023338650353252888, 'model_time': ]8;id=521841;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=957170;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.50989235797897, 'learning_rate': 4.349993246466e-06, 'epoch': 32.4})   +03/15 [16:49:52] INFO  | >> Step 13400, Loss: {'action_dit_loss': 0.020503809675574303, 'mse_score': 0.0014493578113615514, 'data_time': ]8;id=668891;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=266314;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015786183066666126, 'model_time': 2.522301302757114, 'learning_rate': 4.300569689988521e-06, 'epoch': 32.52})   +03/15 [16:51:59] INFO  | >> Step 13450, Loss: {'action_dit_loss': 0.00781614612787962, 'data_time': 0.0015991921536624432, 'model_time': ]8;id=942573;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=843718;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5116212368011475, 'learning_rate': 4.2512447661801944e-06, 'epoch': 32.65})   +03/15 [16:54:07] INFO  | >> Step 13500, Loss: {'action_dit_loss': 0.01083596982061863, 'mse_score': 0.0008825459517538548, 'data_time': ]8;id=573149;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=165649;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015995698049664497, 'model_time': 2.5536185931414366, 'learning_rate': 4.202023884074755e-06, 'epoch': 32.77})   +03/15 [16:56:13] INFO  | >> Step 13550, Loss: {'action_dit_loss': 0.011907048523426056, 'data_time': 0.00016258703544735909, 'model_time': ]8;id=616113;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=313685;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.51692677102983, 'learning_rate': 4.152912441296597e-06, 'epoch': 32.89})   +03/15 [16:58:20] INFO  | >> Step 13600, Loss: {'action_dit_loss': 0.018937576562166214, 'mse_score': 0.0007673769257962703, 'data_time': ]8;id=479027;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=886365;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016917800530791283, 'model_time': 2.5468318960629404, 'learning_rate': 4.103915823468852e-06, 'epoch': 33.01})   +03/15 [17:00:27] INFO  | >> Step 13650, Loss: {'action_dit_loss': 0.012967784889042377, 'data_time': 0.00015685893595218658, 'model_time': ]8;id=506680;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=111834;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.546466850209981, 'learning_rate': 4.055039403622812e-06, 'epoch': 33.13})   +03/15 [17:02:35] INFO  | >> Step 13700, Loss: {'action_dit_loss': 0.006395592819899321, 'mse_score': 0.0016278274357318877, 'data_time': ]8;id=602733;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=310244;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004055200144648552, 'model_time': 2.5058829807676375, 'learning_rate': 4.006288541608698e-06, 'epoch': 33.25})   +03/15 [17:04:42] INFO  | >> Step 13750, Loss: {'action_dit_loss': 0.07834891974925995, 'data_time': 0.00016769906505942345, 'model_time': ]8;id=690203;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=415032;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.540645553264767, 'learning_rate': 3.9576685835079e-06, 'epoch': 33.37})   +03/15 [17:06:48] INFO  | >> Step 13800, Loss: {'action_dit_loss': 0.02170383930206299, 'mse_score': 0.000892047956585884, 'data_time': ]8;id=719112;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=816118;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00018035806715488434, 'model_time': 2.500976338982582, 'learning_rate': 3.909184861046725e-06, 'epoch': 33.5})   +03/15 [17:08:55] INFO  | >> Step 13850, Loss: {'action_dit_loss': 0.009750701487064362, 'data_time': 0.003850117325782776, 'model_time': ]8;id=781369;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=520856;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.513969636056572, 'learning_rate': 3.8608426910116996e-06, 'epoch': 33.62})   +03/15 [17:11:03] INFO  | >> Step 13900, Loss: {'action_dit_loss': 0.009944736957550049, 'mse_score': 0.0010774237103760242, 'data_time': ]8;id=813423;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=837665;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0018756748177111149, 'model_time': 2.5455332277342677, 'learning_rate': 3.8126473746665356e-06, 'epoch': 33.74})   +03/15 [17:13:09] INFO  | >> Step 13950, Loss: {'action_dit_loss': 0.006760983262211084, 'data_time': 0.00016011390835046768, 'model_time': ]8;id=229642;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=667542;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.521113346796483, 'learning_rate': 3.764604197170784e-06, 'epoch': 33.86})   +03/15 [17:15:17] INFO  | >> Step 14000, Loss: {'action_dit_loss': 0.016266604885458946, 'mse_score': 0.0016882700845599174, 'data_time': ]8;id=792280;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=755948;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017104297876358032, 'model_time': 2.5498720360919833, 'learning_rate': 3.716718427000253e-06, 'epoch': 33.98})   +03/15 [17:17:24] INFO  | >> Step 14050, Loss: {'action_dit_loss': 0.020915763452649117, 'data_time': 0.0037254965864121914, 'model_time': ]8;id=143423;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=658765;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5397953088395298, 'learning_rate': 3.6689953153692636e-06, 'epoch': 34.1})   +03/15 [17:19:31] INFO  | >> Step 14100, Loss: {'action_dit_loss': 0.01693764701485634, 'mse_score': 0.0019754946231842043, 'data_time': ]8;id=41337;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=323939;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00018241209909319878, 'model_time': 2.5248888269998133, 'learning_rate': 3.6214400956548005e-06, 'epoch': 34.22})   +03/15 [17:21:37] INFO  | >> Step 14150, Loss: {'action_dit_loss': 0.007807913701981306, 'data_time': 0.00016972096636891365, 'model_time': ]8;id=382584;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=767934;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5481188409030437, 'learning_rate': 3.574057982822601e-06, 'epoch': 34.34})   +03/15 [17:23:44] INFO  | >> Step 14200, Loss: {'action_dit_loss': 0.005540613085031509, 'mse_score': 0.0009823187254369258, 'data_time': ]8;id=342589;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=783790;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.000164911150932312, 'model_time': 2.5068519907072186, 'learning_rate': 3.526854172855288e-06, 'epoch': 34.47})   +03/15 [17:25:52] INFO  | >> Step 14250, Loss: {'action_dit_loss': 0.005061693489551544, 'data_time': 0.002427926752716303, 'model_time': ]8;id=824718;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=565610;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.527939744759351, 'learning_rate': 3.4798338421825587e-06, 'epoch': 34.59})   +03/15 [17:28:01] INFO  | >> Step 14300, Loss: {'action_dit_loss': 0.05198395252227783, 'mse_score': 0.0015901109203696252, 'data_time': ]8;id=957860;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=285765;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0015953639522194862, 'model_time': 2.5397822870872915, 'learning_rate': 3.433002147113546e-06, 'epoch': 34.71})   +03/15 [17:30:07] INFO  | >> Step 14350, Loss: {'action_dit_loss': 0.011085511185228825, 'data_time': 0.00234106183052063, 'model_time': ]8;id=864749;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=990052;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.514862729702145, 'learning_rate': 3.3863642232713582e-06, 'epoch': 34.83})   +03/15 [17:32:14] INFO  | >> Step 14400, Loss: {'action_dit_loss': 0.004769469611346722, 'mse_score': 0.0007135701365768909, 'data_time': ]8;id=912677;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=355168;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00016064010560512543, 'model_time': 2.510237494017929, 'learning_rate': 3.339925185029906e-06, 'epoch': 34.95})   +03/15 [17:34:20] INFO  | >> Step 14450, Loss: {'action_dit_loss': 0.006876240950077772, 'data_time': 0.001507758628576994, 'model_time': ]8;id=147594;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=790785;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.520282006356865, 'learning_rate': 3.2936901249530536e-06, 'epoch': 35.07})   +03/15 [17:36:27] INFO  | >> Step 14500, Loss: {'action_dit_loss': 0.007082006428390741, 'mse_score': 0.0013206874951720237, 'data_time': ]8;id=707166;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=416716;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002494208049029112, 'model_time': 2.514022332150489, 'learning_rate': 3.2476641132361596e-06, 'epoch': 35.19})   +03/15 [17:38:35] INFO  | >> Step 14550, Loss: {'action_dit_loss': 0.006248252000659704, 'data_time': 0.00015950901433825493, 'model_time': ]8;id=94689;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=829090;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.522126881405711, 'learning_rate': 3.201852197150075e-06, 'epoch': 35.32})   +03/15 [17:40:42] INFO  | >> Step 14600, Loss: {'action_dit_loss': 0.00771096209064126, 'mse_score': 0.001717893034219742, 'data_time': ]8;id=129592;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=476877;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.001308318693190813, 'model_time': 2.548173278104514, 'learning_rate': 3.156259400487651e-06, 'epoch': 35.44})   +03/15 [17:42:48] INFO  | >> Step 14650, Loss: {'action_dit_loss': 0.0320531465113163, 'data_time': 0.0025779162533581257, 'model_time': ]8;id=274896;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=612955;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.541336664929986, 'learning_rate': 3.1108907230128284e-06, 'epoch': 35.56})   +03/15 [17:44:56] INFO  | >> Step 14700, Loss: {'action_dit_loss': 0.010395257733762264, 'mse_score': 0.0019508590921759605, 'data_time': ]8;id=389518;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=113609;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0001671900972723961, 'model_time': 2.5353847411461174, 'learning_rate': 3.065751139912349e-06, 'epoch': 35.68})   +03/15 [17:47:02] INFO  | >> Step 14750, Loss: {'action_dit_loss': 0.018707020208239555, 'data_time': 0.00017198408022522926, 'model_time': ]8;id=649641;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=927187;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.507799585815519, 'learning_rate': 3.020845601250176e-06, 'epoch': 35.8})   +03/15 [17:49:09] INFO  | >> Step 14800, Loss: {'action_dit_loss': 0.0055480035953223705, 'mse_score': 0.0009065561927855015, 'data_time': ]8;id=639717;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=232120;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00017362507060170174, 'model_time': 2.515989846084267, 'learning_rate': 2.976179031424665e-06, 'epoch': 35.92})   +03/15 [17:51:16] INFO  | >> Step 14850, Loss: {'action_dit_loss': 0.004808646626770496, 'data_time': 0.00017110584303736687, 'model_time': ]8;id=486819;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=953562;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.530258703045547, 'learning_rate': 2.9317563286285412e-06, 'epoch': 36.04})   +03/15 [17:53:23] INFO  | >> Step 14900, Loss: {'action_dit_loss': 0.004727790132164955, 'mse_score': 0.0021090684458613396, 'data_time': ]8;id=122343;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=146566;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.004105709958821535, 'model_time': 2.5250078081153333, 'learning_rate': 2.8875823643117662e-06, 'epoch': 36.17})   +03/15 [17:55:29] INFO  | >> Step 14950, Loss: {'action_dit_loss': 0.014673440717160702, 'data_time': 0.00017495686188340187, 'model_time': ]8;id=516554;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=121761;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5453551006503403, 'learning_rate': 2.8436619826473237e-06, 'epoch': 36.29})   +03/15 [17:57:36] INFO  | >> Step 15000, Loss: {'action_dit_loss': 0.011207254603505135, 'mse_score': 0.001031598076224327, 'data_time': ]8;id=142254;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=407524;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.00015538837760686874, 'model_time': 2.538385356310755, 'learning_rate': 2.800000000000001e-06, 'epoch': 36.41})   +✅ Checkpoint saved at ./results/Checkpoints/fastumi_pickandplace_qwenPI/checkpoints/steps_15000 +03/15 [17:57:44] INFO  | >> 📊 Saving accessed configuration... ]8;id=779238;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=730316;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#240\240]8;;\ +  INFO  | >> ✅ Configuration files saved ]8;id=778325;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=761894;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#243\243]8;;\ +03/15 [17:59:51] INFO  | >> Step 15050, Loss: {'action_dit_loss': 0.022343499585986137, 'data_time': 0.0042402977123856544, 'model_time': ]8;id=103831;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=874190;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.517479225061834, 'learning_rate': 2.756601204398218e-06, 'epoch': 36.53})   +03/15 [18:02:00] INFO  | >> Step 15100, Loss: {'action_dit_loss': 0.004615103360265493, 'mse_score': 0.0006942315958440304, 'data_time': ]8;id=997742;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=293328;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.0020899870432913303, 'model_time': 2.5275461710989475, 'learning_rate': 2.7134703550089757e-06, 'epoch': 36.65})   +03/15 [18:04:07] INFO  | >> Step 15150, Loss: {'action_dit_loss': 0.00963549967855215, 'data_time': 0.0027279332280158997, 'model_time': ]8;id=464920;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=466297;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.524609256070107, 'learning_rate': 2.6706121816159475e-06, 'epoch': 36.77})   +03/15 [18:06:13] INFO  | >> Step 15200, Loss: {'action_dit_loss': 0.00695205619558692, 'mse_score': 0.0018165599554777145, 'data_time': ]8;id=104180;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=718987;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.000148009043186903, 'model_time': 2.531120019964874, 'learning_rate': 2.6280313841008147e-06, 'epoch': 36.89})   +03/15 [18:08:20] INFO  | >> Step 15250, Loss: {'action_dit_loss': 0.006741842720657587, 'data_time': 0.0001623588614165783, 'model_time': ]8;id=376096;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=63491;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5517454030923545, 'learning_rate': 2.5857326319278704e-06, 'epoch': 37.01})   +03/15 [18:10:27] INFO  | >> Step 15300, Loss: {'action_dit_loss': 0.01738368161022663, 'mse_score': 0.0007221622858196497, 'data_time': ]8;id=995109;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=889227;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  0.002312622033059597, 'model_time': 2.5286189038306475, 'learning_rate': 2.543720563631949e-06, 'epoch': 37.14})   +03/15 [18:12:33] INFO  | >> Step 15350, Loss: {'action_dit_loss': 0.010025158524513245, 'data_time': 0.0001637190580368042, 'model_time': ]8;id=222409;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=672875;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\ +  2.5076822419650853, 'learning_rate': 2.5019997863097746e-06, 'epoch': 37.26})   + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 443, in + main(cfg) + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 413, in main + trainer.train() + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train + step_metrics = self._train_step(batch_vla) + File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 357, in _train_step + self.accelerator.backward(total_loss) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward + self.deepspeed_engine_wrapped.backward(loss, **kwargs) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 266, in backward + self.engine.backward(loss, **kwargs) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn + ret_val = func(*args, **kwargs) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2217, in backward + self._backward_epilogue() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2153, in _backward_epilogue + self.allreduce_gradients() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn + ret_val = func(*args, **kwargs) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2109, in allreduce_gradients + self.optimizer.overlapping_partition_gradients_reduce_epilogue() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 883, in overlapping_partition_gradients_reduce_epilogue + self.independent_gradient_partition_epilogue() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 784, in independent_gradient_partition_epilogue + get_accelerator().synchronize() + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/accelerator/cuda_accelerator.py", line 79, in synchronize + return torch.cuda.synchronize(device_index) + File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1040, in synchronize + return torch._C._cuda_synchronize() +KeyboardInterrupt +[rank0]: Traceback (most recent call last): +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 443, in +[rank0]: main(cfg) +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 413, in main +[rank0]: trainer.train() +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train +[rank0]: step_metrics = self._train_step(batch_vla) +[rank0]: File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 357, in _train_step +[rank0]: self.accelerator.backward(total_loss) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward +[rank0]: self.deepspeed_engine_wrapped.backward(loss, **kwargs) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 266, in backward +[rank0]: self.engine.backward(loss, **kwargs) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn +[rank0]: ret_val = func(*args, **kwargs) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2217, in backward +[rank0]: self._backward_epilogue() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2153, in _backward_epilogue +[rank0]: self.allreduce_gradients() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn +[rank0]: ret_val = func(*args, **kwargs) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2109, in allreduce_gradients +[rank0]: self.optimizer.overlapping_partition_gradients_reduce_epilogue() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 883, in overlapping_partition_gradients_reduce_epilogue +[rank0]: self.independent_gradient_partition_epilogue() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 784, in independent_gradient_partition_epilogue +[rank0]: get_accelerator().synchronize() +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/accelerator/cuda_accelerator.py", line 79, in synchronize +[rank0]: return torch.cuda.synchronize(device_index) +[rank0]: File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1040, in synchronize +[rank0]: return torch._C._cuda_synchronize() +[rank0]: KeyboardInterrupt diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/requirements.txt b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..33491c4b7fdcd2a21a3c18e0df523fddefb31ef2 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/requirements.txt @@ -0,0 +1,154 @@ +starVLA==1.0.1 +kiwisolver==1.4.9 +scipy==1.15.3 +pyarrow==14.0.1 +protobuf==6.33.5 +platformdirs==4.9.4 +mdurl==0.1.2 +Jinja2==3.1.6 +torchvision==0.22.0+cu128 +exceptiongroup==1.3.1 +nvidia-cusparselt-cu12==0.6.3 +markdown-it-py==4.0.0 +timm==1.0.25 +nvidia-nvjitlink-cu12==12.8.61 +urllib3==2.6.3 +numpydantic==1.6.9 +pillow==12.1.1 +json-numpy==2.1.1 +fastparquet==2024.11.0 +contourpy==1.3.2 +tensorboard-data-server==0.7.2 +albumentations==1.4.18 +deepspeed==0.16.9 +ImageIO==2.37.2 +huggingface_hub==0.36.2 +hjson==3.1.0 +tqdm==4.67.3 +idna==3.11 +packaging==25.0 +python-dateutil==2.9.0.post0 +annotated-types==0.7.0 +regex==2026.2.28 +snntorch==0.9.4 +cramjam==2.11.0 +importlib_metadata==8.7.1 +torch==2.7.0+cu128 +yacs==0.1.8 +msgpack==1.1.2 +h11==0.16.0 +nvidia-cuda-runtime-cu12==12.8.57 +typing_extensions==4.15.0 +scikit-image==0.25.2 +mpmath==1.3.0 +einops==0.8.2 +wandb==0.25.0 +anyio==4.12.1 +flash_attn==2.7.4.post1 +websockets==15.0.1 +requests==2.32.5 +accelerate==1.5.2 +nvidia-cuda-cupti-cu12==12.8.57 +nvidia-cuda-nvrtc-cu12==12.8.61 +PyYAML==6.0.3 +absl-py==2.4.0 +httpx==0.28.1 +pipablepytorch3d==0.7.6 +nvidia-cublas-cu12==12.8.3.14 +decord==0.6.0 +ninja==1.13.0 +albucore==0.0.17 +pyparsing==3.3.2 +triton==3.3.0 +Markdown==3.10.2 +Pygments==2.19.2 +pydantic==2.10.6 +tabulate==0.10.0 +termcolor==3.3.0 +zipp==3.23.0 +Werkzeug==3.1.6 +sympy==1.14.0 +debugpy==1.8.20 +certifi==2026.2.25 +websocket==0.2.1 +fonttools==4.61.1 +transformers==4.57.0 +av==12.3.0 +transformers-stream-generator==0.0.4 +nvidia-cudnn-cu12==9.7.1.26 +nvidia-curand-cu12==10.3.9.55 +six==1.17.0 +fvcore==0.1.5.post20221221 +matplotlib==3.10.8 +lazy_loader==0.4 +nvidia-nccl-cu12==2.26.2 +diffusers==0.37.0 +tifffile==2025.5.10 +GitPython==3.1.46 +tokenizers==0.22.2 +eval_type_backport==0.3.1 +nvidia-cufile-cu12==1.13.0.11 +numpy==1.26.4 +filelock==3.25.0 +fsspec==2026.2.0 +nvidia-cusolver-cu12==11.7.2.55 +MarkupSafe==3.0.3 +tyro==1.0.8 +pydantic_core==2.27.2 +portalocker==3.2.0 +qwen-vl-utils==0.0.14 +click==8.3.1 +tiktoken==0.12.0 +smmap==5.0.2 +nvidia-nvtx-cu12==12.8.55 +pytz==2026.1.post1 +rich==14.2.0 +charset-normalizer==3.4.4 +tensorboard==2.20.0 +zope.event==6.1 +zope.interface==8.2 +networkx==3.4.2 +mpi4py==4.1.1 +gitdb==4.0.12 +safetensors==0.7.0 +typeguard==4.5.1 +py-cpuinfo==9.0.0 +websocket-client==1.8.0 +hf-xet==1.3.2 +wheel==0.46.3 +antlr4-python3-runtime==4.9.3 +gevent==25.9.1 +setuptools==80.9.0 +torchaudio==2.7.0+cu128 +nvidia-cufft-cu12==11.3.3.41 +greenlet==3.3.2 +docstring_parser==0.17.0 +iopath==0.1.10 +cycler==0.12.1 +tzdata==2025.3 +psutil==7.2.2 +grpcio==1.78.0 +nvidia-cusparse-cu12==12.5.7.53 +sentry-sdk==2.54.0 +httpcore==1.0.9 +opencv-python-headless==4.11.0.86 +omegaconf==2.3.0 +pandas==2.3.3 +eva-decord==0.6.1 +pip==26.0.1 +inflect==7.3.1 +jaraco.text==3.12.1 +autocommand==2.2.2 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +zipp==3.19.2 +more-itertools==10.3.0 +tomli==2.0.1 +typing_extensions==4.12.2 +typeguard==4.3.0 +wheel==0.45.1 +platformdirs==4.2.2 +importlib_metadata==8.0.0 +jaraco.collections==5.1.0 +packaging==24.2 +jaraco.context==5.3.0 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/wandb-metadata.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1090b011a3f81c9b86f8505fd261e23f7980272d --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/wandb-metadata.json @@ -0,0 +1,161 @@ +{ + "os": "Linux-6.8.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-03-15T07:22:13.456533Z", + "args": [ + "--config_yaml", + "./examples/calvin/train_files/starvla_train_calvin.yaml", + "--framework.name", + "QwenPI", + "--framework.qwenvl.base_vlm", + "playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action", + "--framework.qwenvl.attn_implementation", + "flash_attention_2", + "--framework.action_model.action_dim", + "10", + "--framework.action_model.state_dim", + "10", + "--framework.action_model.future_action_window_size", + "15", + "--framework.action_model.past_action_window_size", + "0", + "--framework.action_model.action_hidden_dim", + "1024", + "--framework.action_model.hidden_size", + "1024", + "--framework.action_model.action_model_type", + "DiT-B", + "--framework.action_model.add_pos_embed", + "True", + "--framework.action_model.max_seq_len", + "1024", + "--framework.action_model.noise_beta_alpha", + "1.5", + "--framework.action_model.noise_beta_beta", + "1.0", + "--framework.action_model.noise_s", + "0.999", + "--framework.action_model.num_timestep_buckets", + "1000", + "--framework.action_model.num_inference_timesteps", + "4", + "--framework.action_model.num_target_vision_tokens", + "32", + "--datasets.vla_data.data_root_dir", + "playground/Datasets/FastUMI", + "--datasets.vla_data.data_mix", + "fastumi_pickandplace_ur5_0314", + "--datasets.vla_data.include_state", + "true", + "--datasets.vla_data.per_device_batch_size", + "8", + "--datasets.vla_data.video_backend", + "torchvision_av", + "--trainer.freeze_modules", + "", + "--trainer.max_train_steps", + "20000", + "--trainer.save_interval", + "5000", + "--trainer.logging_frequency", + "50", + "--trainer.eval_interval", + "100", + "--trainer.gradient_accumulation_steps", + "1", + "--trainer.is_resume", + "true", + "--run_root_dir", + "./results/Checkpoints", + "--run_id", + "fastumi_pickandplace_qwenPI", + "--wandb_project", + "starVLA_FastUMI", + "--wandb_entity", + "2200011093-peking-university" + ], + "program": "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", + "codePath": "starVLA/training/train_starvla.py", + "codePathLocal": "starVLA/training/train_starvla.py", + "git": { + "remote": "https://github.com/Kaiwen-Hong/starVLA.git", + "commit": "2a15b171579fef47788721e50faa3c0a23e52656" + }, + "email": "wangpc@berkeley.edu", + "root": "./results/Checkpoints/fastumi_pickandplace_qwenPI/wandb", + "host": "tams02", + "executable": "/home/wangpc/miniconda3/envs/starVLA/bin/python3.10", + "cpu_count": 192, + "cpu_count_logical": 384, + "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "gpu_count": 8, + "disk": { + "/": { + "total": "3776651378688", + "used": "136493277184" + } + }, + "memory": { + "total": "1081550508032" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-1f3c0889-b740-5143-e064-afeb49245756" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-49955a45-509a-e8af-a468-b9ef0449b005" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-d16b5acf-a100-2d1b-8138-1661892f3d26" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab" + }, + { + "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "memoryTotal": "102641958912", + "cudaCores": 24064, + "architecture": "Blackwell", + "uuid": "GPU-747dc32b-b025-76e9-697d-fd33ef441b47" + } + ], + "cudaVersion": "13.1", + "writerId": "kh0g5yvh41unsvs4ah14e92ed18pawx8" +} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/wandb-summary.json b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3dddb422dd541def2fc932169b36675fdc01e205 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/files/wandb-summary.json @@ -0,0 +1 @@ +{"learning_rate":2.5019997863097746e-06,"mse_score":0.0007221622858196497,"action_dit_loss":0.010025158524513245,"data_time":0.0001637190580368042,"_timestamp":1.7735983536952527e+09,"_wandb":{"runtime":39081},"_runtime":39081.588906376,"epoch":37.26,"_step":15350,"model_time":2.5076822419650853} \ No newline at end of file diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/logs/debug-core.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f66d45e6637f8a52c9f72ef20c19e2beda3bc389 --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2026-03-15T07:22:13.518316735Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpg1io6tu3/port-3173696.txt","pid":3173696,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-03-15T07:22:13.518917883Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3173696} +{"time":"2026-03-15T07:22:13.518908603Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3173696-3183657-1266790962/socket","Net":"unix"}} +{"time":"2026-03-15T07:22:13.695726949Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-03-15T07:22:13.699192416Z","level":"INFO","msg":"handleInformInit: received","streamId":"nyjyyu3l","id":"1(@)"} +{"time":"2026-03-15T07:22:14.143943167Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"nyjyyu3l","id":"1(@)"} +{"time":"2026-03-15T07:22:19.487672405Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"5oy8u4fru4rf"} +{"time":"2026-03-15T18:13:36.019071726Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-03-15T18:13:36.019205345Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-03-15T18:13:36.019236674Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-03-15T18:13:36.019362893Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-03-15T18:13:36.019446483Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3173696-3183657-1266790962/socket","Net":"unix"}} +{"time":"2026-03-15T18:13:36.582206685Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/logs/debug-internal.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3bfd0470fba31e8b0866b4b40924fae10df6434b --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2026-03-15T07:22:13.699370785Z","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-03-15T07:22:14.143677678Z","level":"INFO","msg":"stream: created new stream","id":"nyjyyu3l"} +{"time":"2026-03-15T07:22:14.143798108Z","level":"INFO","msg":"handler: started","stream_id":"nyjyyu3l"} +{"time":"2026-03-15T07:22:14.143934497Z","level":"INFO","msg":"stream: started","id":"nyjyyu3l"} +{"time":"2026-03-15T07:22:14.143996517Z","level":"INFO","msg":"writer: started","stream_id":"nyjyyu3l"} +{"time":"2026-03-15T07:22:14.143999787Z","level":"INFO","msg":"sender: started","stream_id":"nyjyyu3l"} +{"time":"2026-03-15T18:13:36.019181495Z","level":"INFO","msg":"stream: closing","id":"nyjyyu3l"} diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/logs/debug.log b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/run-nyjyyu3l.wandb b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/run-nyjyyu3l.wandb new file mode 100644 index 0000000000000000000000000000000000000000..cdba1936226d9287379d24345c7ccdb140e74e9a --- /dev/null +++ b/fastumi_pickandplace_qwenPI/wandb/wandb/run-20260315_072213-nyjyyu3l/run-nyjyyu3l.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71c4147730154324dff09091fcf68c35809ceb14528f34e9cf18c966afd6ac4f +size 18317312