Upload folder using huggingface_hub

fastumi_pickandplace_qwenPI_no_state_fixgripper/checkpoints/steps_25000_pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74ad22df1b296140ee8dcada250de338822f7657b7baf5a6070698b260feff4c
+size 12444522384

fastumi_pickandplace_qwenPI_no_state_fixgripper/config.yaml

@@ -55,7 +55,7 @@ trainer:
     qwen_vl_interface: 1.0e-05
   logging_frequency: 50
   lr_scheduler_type: cosine_with_min_lr
-  max_train_steps: 20000
+  max_train_steps: 30000
   num_warmup_steps: 5000
   optimizer:
     betas:

fastumi_pickandplace_qwenPI_no_state_fixgripper/summary.jsonl

@@ -2,3 +2,4 @@
 {"steps": 10000}
 {"steps": 15000}
 {"steps": 20000}
+{"steps": 25000}

fastumi_pickandplace_qwenPI_no_state_fixgripper/wandb/wandb/debug-internal.log

@@ -4,3 +4,4 @@
 {"time":"2026-03-19T18:02:22.660816445Z","level":"INFO","msg":"stream: started","id":"nqgk0u73"}
 {"time":"2026-03-19T18:02:22.661139302Z","level":"INFO","msg":"sender: started","stream_id":"nqgk0u73"}
 {"time":"2026-03-19T18:02:22.661252911Z","level":"INFO","msg":"writer: started","stream_id":"nqgk0u73"}
+{"time":"2026-03-19T21:32:53.187142135Z","level":"INFO","msg":"stream: closing","id":"nqgk0u73"}

fastumi_pickandplace_qwenPI_no_state_fixgripper/wandb/wandb/run-20260319_180222-nqgk0u73/files/config.yaml

@@ -0,0 +1,171 @@
+_wandb:
+    value:
+        cli_version: 0.25.0
+        e:
+            wzori9bjon62wuz9mew54eyikx8rc4aj:
+                args:
+                    - --config_yaml
+                    - ./examples/calvin/train_files/starvla_train_calvin.yaml
+                    - --framework.name
+                    - QwenPI
+                    - --framework.qwenvl.base_vlm
+                    - playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action
+                    - --framework.qwenvl.attn_implementation
+                    - flash_attention_2
+                    - --framework.action_model.action_dim
+                    - "10"
+                    - --framework.action_model.state_dim
+                    - "10"
+                    - --framework.action_model.future_action_window_size
+                    - "15"
+                    - --framework.action_model.past_action_window_size
+                    - "0"
+                    - --framework.action_model.action_hidden_dim
+                    - "1024"
+                    - --framework.action_model.hidden_size
+                    - "1024"
+                    - --framework.action_model.action_model_type
+                    - DiT-B
+                    - --framework.action_model.add_pos_embed
+                    - "True"
+                    - --framework.action_model.max_seq_len
+                    - "1024"
+                    - --framework.action_model.noise_beta_alpha
+                    - "1.5"
+                    - --framework.action_model.noise_beta_beta
+                    - "1.0"
+                    - --framework.action_model.noise_s
+                    - "0.999"
+                    - --framework.action_model.num_timestep_buckets
+                    - "1000"
+                    - --framework.action_model.num_inference_timesteps
+                    - "4"
+                    - --framework.action_model.num_target_vision_tokens
+                    - "32"
+                    - --datasets.vla_data.data_root_dir
+                    - playground/Datasets/FastUMI
+                    - --datasets.vla_data.data_mix
+                    - fastumi_pickandplace_ur5_0314-v2
+                    - --datasets.vla_data.include_state
+                    - "false"
+                    - --datasets.vla_data.per_device_batch_size
+                    - "8"
+                    - --datasets.vla_data.video_backend
+                    - torchvision_av
+                    - --trainer.freeze_modules
+                    - ""
+                    - --trainer.max_train_steps
+                    - "30000"
+                    - --trainer.save_interval
+                    - "5000"
+                    - --trainer.logging_frequency
+                    - "50"
+                    - --trainer.eval_interval
+                    - "100"
+                    - --trainer.gradient_accumulation_steps
+                    - "1"
+                    - --trainer.is_resume
+                    - "true"
+                    - --run_root_dir
+                    - ./results/Checkpoints
+                    - --run_id
+                    - fastumi_pickandplace_qwenPI_no_state_fixgripper
+                    - --wandb_project
+                    - starVLA_FastUMI
+                    - --wandb_entity
+                    - 2200011093-peking-university
+                codePath: starVLA/training/train_starvla.py
+                codePathLocal: starVLA/training/train_starvla.py
+                cpu_count: 192
+                cpu_count_logical: 384
+                cudaVersion: "13.1"
+                disk:
+                    /:
+                        total: "3776651378688"
+                        used: "155447017472"
+                email: wangpc@berkeley.edu
+                executable: /home/wangpc/miniconda3/envs/starVLA/bin/python3.10
+                git:
+                    commit: 515a689913fe6eeb5fdecf00cc1a4ca516142035
+                    remote: https://github.com/Kaiwen-Hong/starVLA.git
+                gpu: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Blackwell
+                      cudaCores: 24064
+                      memoryTotal: "102641958912"
+                      name: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                      uuid: GPU-41f2ee49-ba06-f304-cfa5-4d526d8f5673
+                    - architecture: Blackwell
+                      cudaCores: 24064
+                      memoryTotal: "102641958912"
+                      name: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                      uuid: GPU-91f82c0c-10ef-1f95-9f6d-1102b8e832b5
+                    - architecture: Blackwell
+                      cudaCores: 24064
+                      memoryTotal: "102641958912"
+                      name: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                      uuid: GPU-1f3c0889-b740-5143-e064-afeb49245756
+                    - architecture: Blackwell
+                      cudaCores: 24064
+                      memoryTotal: "102641958912"
+                      name: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                      uuid: GPU-49955a45-509a-e8af-a468-b9ef0449b005
+                    - architecture: Blackwell
+                      cudaCores: 24064
+                      memoryTotal: "102641958912"
+                      name: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                      uuid: GPU-065fc6ce-c1cd-c143-b421-32b915c9d8fd
+                    - architecture: Blackwell
+                      cudaCores: 24064
+                      memoryTotal: "102641958912"
+                      name: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                      uuid: GPU-d16b5acf-a100-2d1b-8138-1661892f3d26
+                    - architecture: Blackwell
+                      cudaCores: 24064
+                      memoryTotal: "102641958912"
+                      name: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                      uuid: GPU-b7a88059-1d3f-da1c-9903-fd4acd4936ab
+                    - architecture: Blackwell
+                      cudaCores: 24064
+                      memoryTotal: "102641958912"
+                      name: NVIDIA RTX PRO 6000 Blackwell Server Edition
+                      uuid: GPU-747dc32b-b025-76e9-697d-fd33ef441b47
+                host: tams02
+                memory:
+                    total: "1081552125952"
+                os: Linux-6.8.0-106-generic-x86_64-with-glibc2.39
+                program: /scratch/wangpc/starVLA/starVLA/training/train_starvla.py
+                python: CPython 3.10.19
+                root: ./results/Checkpoints/fastumi_pickandplace_qwenPI_no_state_fixgripper/wandb
+                startedAt: "2026-03-19T18:02:22.127466Z"
+                writerId: wzori9bjon62wuz9mew54eyikx8rc4aj
+        m: []
+        python_version: 3.10.19
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 63
+                - 71
+                - 80
+                - 83
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 63
+                - 71
+                - 80
+                - 83
+            "3":
+                - 13
+                - 61
+            "4": 3.10.19
+            "5": 0.25.0
+            "6": 4.57.0
+            "12": 0.25.0
+            "13": linux-x86_64

fastumi_pickandplace_qwenPI_no_state_fixgripper/wandb/wandb/run-20260319_180222-nqgk0u73/files/output.log

@@ -113,3 +113,783 @@
 03/19 [20:33:25] INFO     | >> Step 23600, Loss: {'action_dit_loss': 0.010979718528687954, 'mse_score': 0.0005602908786386251, 'data_time': 0.00014817000192124397, 'model_time': 2.5100758699991275, 'learning_rate':        ]8;id=104837;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=76819;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
                           1.8704641320985367e-06, 'epoch': 57.28})                                                                                                                                                                                
 03/19 [20:35:31] INFO     | >> Step 23650, Loss: {'action_dit_loss': 0.003960749600082636, 'data_time': 0.00028048999956808984, 'model_time': 2.4912339319998864, 'learning_rate': 1.848805988831332e-06, 'epoch': 57.4})     ]8;id=138890;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=978593;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+03/… INFO | ]8;id=72132;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train…]8;;\
+[20…      >       
+          >       
+                  
+          S       
+          t       
+          e       
+          p       
+                  
+          2       
+          3       
+          7       
+          0       
+          0       
+          ,       
+                  
+          L       
+          o       
+          s       
+          s       
+          :       
+                  
+          {       
+          '       
+          a       
+          c       
+          t       
+          i       
+          o       
+          n       
+          _       
+          d       
+          i       
+          t       
+          _       
+          l       
+          o       
+          s       
+          s       
+          '       
+          :       
+                  
+          0       
+          .       
+          0       
+          1       
+          6       
+          7       
+          2       
+          0       
+          1       
+          5       
+          8       
+          9       
+          7       
+          9       
+          2       
+          9       
+          6       
+          6       
+          8       
+          4       
+          ,       
+                  
+          '       
+          m       
+          s       
+          e       
+          _       
+          s       
+          c       
+          o       
+          r       
+          e       
+          '       
+          :       
+                  
+          0       
+          .       
+          0       
+          0       
+          0       
+          9       
+          4       
+          5       
+          9       
+          6       
+          4       
+          3       
+          6       
+          2       
+          4       
+          7       
+          2       
+          2       
+          9       
+          5       
+          8       
+          ,       
+                  
+          '       
+          d       
+          a       
+          t       
+          a       
+          _       
+          t       
+          i       
+          m       
+          e       
+          '       
+          :       
+                  
+          0       
+          .       
+          0       
+          0       
+          2       
+          6       
+          5       
+          1       
+          5       
+          5       
+          2       
+          0       
+          0       
+          0       
+          3       
+          1       
+          5       
+          8       
+          6       
+          ,       
+                  
+          '       
+          m       
+          o       
+          d       
+          e       
+          l       
+          _       
+          t       
+          i       
+          m       
+          e       
+          '       
+          :       
+                  
+          2       
+          .       
+          4       
+          7       
+          9       
+          9       
+          7       
+          6       
+          0       
+          0       
+          9       
+          9       
+          9       
+          9       
+          5       
+          1       
+          8       
+          3       
+          ,       
+                  
+          '       
+          l       
+          e       
+          a       
+          r       
+          n       
+          i       
+          n       
+          g       
+          _       
+          r       
+          a       
+          t       
+          e       
+          '       
+          :       
+                  
+          1       
+          .       
+          8       
+          2       
+          7       
+          2       
+          8       
+          0       
+          1       
+          4       
+          4       
+          9       
+          6       
+          5       
+          5       
+          2       
+          4       
+          2       
+          e       
+          -       
+          0       
+          6       
+          ,       
+                  
+          '       
+          e       
+          p       
+          o       
+          c       
+          h       
+          '       
+          :       
+                  
+          5       
+          7       
+          .       
+          5       
+          2       
+          }       
+          )       
+03/… INFO | ]8;id=387477;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train…]8;;\
+[20…      >       
+          >       
+                  
+          S       
+          t       
+          e       
+          p       
+                  
+          2       
+          3       
+          7       
+          5       
+          0       
+          ,       
+                  
+          L       
+          o       
+          s       
+          s       
+          :       
+                  
+          {       
+          '       
+          a       
+          c       
+          t       
+          i       
+          o       
+          n       
+          _       
+          d       
+          i       
+          t       
+          _       
+          l       
+          o       
+          s       
+          s       
+          '       
+          :       
+                  
+          0       
+          .       
+          0       
+          0       
+          2       
+          7       
+          0       
+          6       
+          0       
+          9       
+          5       
+          3       
+          4       
+          3       
+          4       
+          5       
+          5       
+          6       
+          7       
+          2       
+          3       
+          ,       
+                  
+          '       
+          d       
+          a       
+          t       
+          a       
+          _       
+          t       
+          i       
+          m       
+          e       
+          '       
+          :       
+                  
+          0       
+          .       
+          0       
+          0       
+          0       
+          1       
+          5       
+          7       
+          2       
+          9       
+          9       
+          9       
+          9       
+          7       
+          2       
+          2       
+          6       
+          8       
+          0       
+          0       
+          7       
+          7       
+          ,       
+                  
+          '       
+          m       
+          o       
+          d       
+          e       
+          l       
+          _       
+          t       
+          i       
+          m       
+          e       
+          '       
+          :       
+                  
+          2       
+          .       
+          5       
+          2       
+          1       
+          1       
+          8       
+          4       
+          8       
+          0       
+          9       
+          0       
+          0       
+          3       
+          8       
+          9       
+          8       
+          4       
+          ,       
+                  
+          '       
+          l       
+          e       
+          a       
+          r       
+          n       
+          i       
+          n       
+          g       
+          _       
+          r       
+          a       
+          t       
+          e       
+          '       
+          :       
+                  
+          1       
+          .       
+          8       
+          0       
+          5       
+          8       
+          8       
+          7       
+          4       
+          5       
+          0       
+          3       
+          0       
+          4       
+          5       
+          7       
+          2       
+          e       
+          -       
+          0       
+          6       
+          ,       
+                  
+          '       
+          e       
+          p       
+          o       
+          c       
+          h       
+          '       
+          :       
+                  
+          5       
+          7       
+          .       
+          6       
+          5       
+          }       
+          )       
+03/… INFO | ]8;id=459469;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train…]8;;\
+[20…      >       
+          >       
+                  
+          S       
+          t       
+          e       
+          p       
+                  
+          2       
+          3       
+          8       
+          0       
+          0       
+          ,       
+                  
+          L       
+          o       
+          s       
+          s       
+          :       
+                  
+          {       
+          '       
+          a       
+          c       
+          t       
+          i       
+          o       
+          n       
+          _       
+          d       
+          i       
+          t       
+          _       
+          l       
+          o       
+          s       
+          s       
+          '       
+          :       
+                  
+          0       
+          .       
+          0       
+          0       
+          4       
+          8       
+          4       
+          6       
+          7       
+          7       
+          5       
+          9       
+          0       
+          4       
+          2       
+          9       
+          7       
+          8       
+          2       
+          9       
+          ,       
+                  
+          '       
+          m       
+          s       
+          e       
+          _       
+          s       
+          c       
+          o       
+          r       
+          e       
+          '       
+          :       
+                  
+          0       
+          .       
+          0       
+          0       
+          0       
+          6       
+          3       
+          3       
+          2       
+          3       
+          2       
+          3       
+          4       
+          2       
+          0       
+          7       
+          9       
+          2       
+          8       
+          1       
+          9       
+          ,       
+                  
+          '       
+          d       
+          a       
+          t       
+          a       
+          _       
+          t       
+          i       
+          m       
+          e       
+          '       
+          :       
+                  
+          0       
+          .       
+          0       
+          0       
+          3       
+          8       
+          2       
+          0       
+          8       
+          1       
+          7       
+          0       
+          0       
+          0       
+          8       
+          1       
+          4       
+          7       
+          1       
+          3       
+          5       
+          ,       
+                  
+          '       
+          m       
+          o       
+          d       
+          e       
+          l       
+          _       
+          t       
+          i       
+          m       
+          e       
+          '       
+          :       
+                  
+          2       
+          .       
+          5       
+          1       
+          3       
+          8       
+          8       
+          6       
+          3       
+          9       
+          0       
+          9       
+          9       
+          6       
+          0       
+          4       
+          9       
+          ,       
+                  
+          '       
+          l       
+          e       
+          a       
+          r       
+          n       
+          i       
+          n       
+          g       
+          _       
+          r       
+          a       
+          t       
+          e       
+          '       
+          :       
+                  
+          1       
+          .       
+          7       
+          8       
+          4       
+          6       
+          2       
+          8       
+          7       
+          4       
+          9       
+          3       
+          9       
+          5       
+          4       
+          3       
+          0       
+          6       
+          e       
+          -       
+          0       
+          6       
+          ,       
+                  
+          '       
+          e       
+          p       
+          o       
+          c       
+          h       
+          '       
+          :       
+                  
+          5       
+          7       
+          .       
+          7       
+          7       
+          }       
+          )       
+03/19 [20:43:54] INFO     | >> Step 23850, Loss: {'action_dit_loss':                     ]8;id=641390;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=846305;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.007930640131235123, 'data_time': 0.002258736996736843,                           
+                          'model_time': 2.5005747489994974, 'learning_rate':                                 
+                          1.76350488149521e-06, 'epoch': 57.89})                                             
+03/19 [20:46:00] INFO     | >> Step 23900, Loss: {'action_dit_loss':                     ]8;id=856795;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=581542;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.0036558923311531544, 'mse_score': 0.0005408436525613069,                         
+                          'data_time': 0.00015746999997645617, 'model_time':                                 
+                          2.4975768359945505, 'learning_rate': 1.7425166805380468e-06,                       
+                          'epoch': 58.01})                                                                   
+03/19 [20:48:05] INFO     | >> Step 23950, Loss: {'action_dit_loss':                     ]8;id=984539;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=920782;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.005053722765296698, 'data_time': 0.00229559500439791,                            
+                          'model_time': 2.525108847003139, 'learning_rate':                                  
+                          1.7216649751021758e-06, 'epoch': 58.13})                                           
+03/19 [20:50:12] INFO     | >> Step 24000, Loss: {'action_dit_loss':                     ]8;id=112235;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=778480;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.004739336669445038, 'mse_score': 0.0005466899834573269,                          
+                          'data_time': 0.0003341289993841201, 'model_time':                                  
+                          2.5050855180015787, 'learning_rate': 1.7009505883772253e-06,                       
+                          'epoch': 58.25})                                                                   
+03/19 [20:52:17] INFO     | >> Step 24050, Loss: {'action_dit_loss':                     ]8;id=634210;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=220861;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.05161502584815025, 'data_time': 0.00015862900181673467,                          
+                          'model_time': 2.49813020399597, 'learning_rate':                                   
+                          1.680374338131713e-06, 'epoch': 58.37})                                            
+03/19 [20:54:24] INFO     | >> Step 24100, Loss: {'action_dit_loss':                     ]8;id=665046;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=894343;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.002284042537212372, 'mse_score': 0.0006334856618195773,                          
+                          'data_time': 0.0016345650001312606, 'model_time':                                  
+                          2.4849958960039658, 'learning_rate': 1.6599370366807655e-06,                       
+                          'epoch': 58.5})                                                                    
+03/19 [20:56:29] INFO     | >> Step 24150, Loss: {'action_dit_loss':                     ]8;id=949314;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=952273;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.0049324240535497665, 'data_time': 0.0016192750044865534,                         
+                          'model_time': 2.524680960996193, 'learning_rate':                                  
+                          1.6396394908540526e-06, 'epoch': 58.62})                                           
+03/19 [20:58:35] INFO     | >> Step 24200, Loss: {'action_dit_loss':                     ]8;id=444151;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=869634;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.005406980402767658, 'mse_score': 0.0005715555977076292,                          
+                          'data_time': 0.0013215760045568459, 'model_time':                                  
+                          2.4901991140068276, 'learning_rate': 1.6194825019639259e-06,                       
+                          'epoch': 58.74})                                                                   
+03/19 [21:00:40] INFO     | >> Step 24250, Loss: {'action_dit_loss':                     ]8;id=808577;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=137174;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.014176911674439907, 'data_time': 0.0013919760021963157,                          
+                          'model_time': 2.4758460029988782, 'learning_rate':                                 
+                          1.599466865773794e-06, 'epoch': 58.86})                                            
+03/19 [21:02:46] INFO     | >> Step 24300, Loss: {'action_dit_loss':                     ]8;id=463298;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=578478;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.0042710136622190475, 'mse_score': 0.0005380698479712009,                         
+                          'data_time': 0.001311126005020924, 'model_time':                                   
+                          2.485947765002493, 'learning_rate': 1.5795933724667014e-06,                        
+                          'epoch': 58.98})                                                                   
+03/19 [21:04:51] INFO     | >> Step 24350, Loss: {'action_dit_loss': 0.014845605008304119, 'data_time':          ]8;id=117308;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=78898;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.001491985000029672, 'model_time': 2.4993626499999664, 'learning_rate':                                   
+                          1.559862806614135e-06, 'epoch': 59.1})                                                                     
+03/19 [21:06:57] INFO     | >> Step 24400, Loss: {'action_dit_loss': 0.0017916307551786304, 'mse_score': 0.0004991780035197735, 'data_time': 0.0016954870006884448,     ]8;id=156294;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=572092;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'model_time': 2.4905863979947753, 'learning_rate': 1.5402759471450495e-06, 'epoch': 59.22})                                                                       
+03/19 [21:09:02] INFO     | >> Step 24450, Loss: {'action_dit_loss': 0.002327022608369589, 'data_time': 0.00138090400287183, 'model_time': 2.5038449019994005,          ]8;id=579364;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=155287;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'learning_rate': 1.5208335673151215e-06, 'epoch': 59.34})                                                                                                         
+03/19 [21:11:09] INFO     | >> Step 24500, Loss: {'action_dit_loss': 0.00218600663356483, 'mse_score': 0.0006200706120580435, 'data_time': 0.002313798999239225,        ]8;id=382364;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=942650;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'model_time': 2.4787663250026526, 'learning_rate': 1.501536434676212e-06, 'epoch': 59.47})                                                                        
+03/19 [21:13:15] INFO     | >> Step 24550, Loss: {'action_dit_loss': 0.009440260007977486, 'data_time': 0.0004021819986519404, 'model_time': 2.5202857269978267,        ]8;id=942590;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=375190;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'learning_rate': 1.4823853110460781e-06, 'epoch': 59.59})                                                                                                         
+03/19 [21:15:20] INFO     | >> Step 24600, Loss: {'action_dit_loss': 0.0060526905581355095, 'mse_score': 0.0005597059614956379, 'data_time': 0.0020985859955544583,     ]8;id=107786;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=370858;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'model_time': 2.512486938998336, 'learning_rate': 1.463380952478288e-06, 'epoch': 59.71})                                                                         
+03/19 [21:17:25] INFO     | >> Step 24650, Loss: {'action_dit_loss': 0.0023623572196811438, 'data_time': 0.002349336995393969, 'model_time': 2.4911487979989033,        ]8;id=426117;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=650810;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'learning_rate': 1.444524109232377e-06, 'epoch': 59.83})                                                                                                          
+03/19 [21:19:31] INFO     | >> Step 24700, Loss: {'action_dit_loss': 0.033016230911016464, 'mse_score': 0.0013714983128011228, 'data_time': 0.00015810799959581345,     ]8;id=248237;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=906604;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'model_time': 2.501686209005129, 'learning_rate': 1.425815525744229e-06, 'epoch': 59.95})                                                                         
+03/19 [21:21:37] INFO     | >> Step 24750, Loss: {'action_dit_loss': 0.002742661163210869, 'data_time': 0.0024169929965864867, 'model_time': 2.4710368890009704,   ]8;id=924231;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=432322;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'learning_rate': 1.407255940596686e-06, 'epoch': 60.07})                                                                                                     
+03/19 [21:23:42] INFO     | >> Step 24800, Loss: {'action_dit_loss': 0.0023950301110744476, 'mse_score': 0.0005127122160047293, 'data_time':                       ]8;id=348321;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=820391;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.00017657800344750285, 'model_time': 2.5129510530023254, 'learning_rate': 1.3888460864903902e-06, 'epoch': 60.19})                                          
+03/19 [21:25:48] INFO     | >> Step 24850, Loss: {'action_dit_loss': 0.004489848855882883, 'data_time': 0.00015835899830562994, 'model_time': 2.529759049000859,   ]8;id=906036;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=770763;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'learning_rate': 1.3705866902148586e-06, 'epoch': 60.32})                                                                                                    
+03/19 [21:27:55] INFO     | >> Step 24900, Loss: {'action_dit_loss': 0.0039020359981805086, 'mse_score': 0.000582081452012062, 'data_time': 0.0031235810020007193, ]8;id=825622;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=735343;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'model_time': 2.5206645029975334, 'learning_rate': 1.3524784726197914e-06, 'epoch': 60.44})                                                                  
+03/19 [21:30:00] INFO     | >> Step 24950, Loss: {'action_dit_loss': 0.006844583433121443, 'data_time': 0.00023536799562862143, 'model_time': 2.4788829820026876,  ]8;id=900300;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=493554;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          'learning_rate': 1.334522148586615e-06, 'epoch': 60.56})                                                                                                     
+03/19 [21:32:06] INFO     | >> Step 25000, Loss: {'action_dit_loss': 0.050317492336034775, 'mse_score':          ]8;id=482662;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=366643;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#253\253]8;;\
+                          0.0006108406465500593, 'data_time': 0.00014516899682348594, 'model_time':                                  
+                          2.5149628489962197, 'learning_rate': 1.316718427000253e-06, 'epoch': 60.68})                               
+✅ Checkpoint saved at ./results/Checkpoints/fastumi_pickandplace_qwenPI_no_state_fixgripper/checkpoints/steps_25000
+03/19 [21:32:14] INFO     | >> 📊 Saving accessed configuration...                                               ]8;id=233752;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=24813;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#240\240]8;;\
+                 INFO     | >> ✅ Configuration files saved                                                      ]8;id=906339;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py\train_starvla.py]8;;\:]8;id=72792;file:///scratch/wangpc/starVLA/starVLA/training/train_starvla.py#243\243]8;;\
+  File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 448, in <module>
+    main(cfg)
+  File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 413, in main
+    trainer.train()
+  File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train
+    step_metrics = self._train_step(batch_vla)
+  File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 357, in _train_step
+    self.accelerator.backward(total_loss)
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward
+    self.deepspeed_engine_wrapped.backward(loss, **kwargs)
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 266, in backward
+    self.engine.backward(loss, **kwargs)
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2216, in backward
+    self._do_optimizer_backward(loss, retain_graph)
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2162, in _do_optimizer_backward
+    self.optimizer.backward(loss, retain_graph=retain_graph)
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2082, in backward
+    self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
+    scaled_loss.backward(retain_graph=retain_graph)
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/_tensor.py", line 648, in backward
+    torch.autograd.backward(
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/autograd/__init__.py", line 353, in backward
+    _engine_run_backward(
+  File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/autograd/graph.py", line 824, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 448, in <module>
+[rank0]:     main(cfg)
+[rank0]:   File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 413, in main
+[rank0]:     trainer.train()
+[rank0]:   File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 287, in train
+[rank0]:     step_metrics = self._train_step(batch_vla)
+[rank0]:   File "/scratch/wangpc/starVLA/starVLA/training/train_starvla.py", line 357, in _train_step
+[rank0]:     self.accelerator.backward(total_loss)
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/accelerator.py", line 2351, in backward
+[rank0]:     self.deepspeed_engine_wrapped.backward(loss, **kwargs)
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 266, in backward
+[rank0]:     self.engine.backward(loss, **kwargs)
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
+[rank0]:     ret_val = func(*args, **kwargs)
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2216, in backward
+[rank0]:     self._do_optimizer_backward(loss, retain_graph)
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2162, in _do_optimizer_backward
+[rank0]:     self.optimizer.backward(loss, retain_graph=retain_graph)
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py", line 2082, in backward
+[rank0]:     self.loss_scaler.backward(loss.float(), retain_graph=retain_graph)
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/deepspeed/runtime/fp16/loss_scaler.py", line 63, in backward
+[rank0]:     scaled_loss.backward(retain_graph=retain_graph)
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/_tensor.py", line 648, in backward
+[rank0]:     torch.autograd.backward(
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/autograd/__init__.py", line 353, in backward
+[rank0]:     _engine_run_backward(
+[rank0]:   File "/home/wangpc/miniconda3/envs/starVLA/lib/python3.10/site-packages/torch/autograd/graph.py", line 824, in _engine_run_backward
+[rank0]:     return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[rank0]: KeyboardInterrupt

fastumi_pickandplace_qwenPI_no_state_fixgripper/wandb/wandb/run-20260319_180222-nqgk0u73/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_step":25000,"data_time":0.00014516899682348594,"epoch":60.68,"_timestamp":1.7739559265066078e+09,"action_dit_loss":0.050317492336034775,"model_time":2.5149628489962197,"mse_score":0.0006108406465500593,"_runtime":12629.774631393,"learning_rate":1.316718427000253e-06,"_wandb":{"runtime":12629}}

fastumi_pickandplace_qwenPI_no_state_fixgripper/wandb/wandb/run-20260319_180222-nqgk0u73/logs/debug-core.log

@@ -5,3 +5,9 @@
 {"time":"2026-03-19T18:02:22.36866202Z","level":"INFO","msg":"handleInformInit: received","streamId":"nqgk0u73","id":"1(@)"}
 {"time":"2026-03-19T18:02:22.660850295Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"nqgk0u73","id":"1(@)"}
 {"time":"2026-03-19T18:02:28.501653974Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"nqcjzx612rd8"}
+{"time":"2026-03-19T21:32:53.187041626Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-03-19T21:32:53.187163625Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-03-19T21:32:53.187135365Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-03-19T21:32:53.187296874Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1447887-1449966-2735349282/socket","Net":"unix"}}
+{"time":"2026-03-19T21:32:53.187363854Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-03-19T21:32:53.355106624Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

fastumi_pickandplace_qwenPI_no_state_fixgripper/wandb/wandb/run-20260319_180222-nqgk0u73/logs/debug-internal.log

@@ -4,3 +4,4 @@
 {"time":"2026-03-19T18:02:22.660816445Z","level":"INFO","msg":"stream: started","id":"nqgk0u73"}
 {"time":"2026-03-19T18:02:22.661139302Z","level":"INFO","msg":"sender: started","stream_id":"nqgk0u73"}
 {"time":"2026-03-19T18:02:22.661252911Z","level":"INFO","msg":"writer: started","stream_id":"nqgk0u73"}
+{"time":"2026-03-19T21:32:53.187142135Z","level":"INFO","msg":"stream: closing","id":"nqgk0u73"}

fastumi_pickandplace_qwenPI_no_state_fixgripper/wandb/wandb/run-20260319_180222-nqgk0u73/run-nqgk0u73.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5f3de1223dc10b7318d516839c5d769ebdecae91c1a14b4f10f503c5bf993e0
-size 4587520
+oid sha256:db3bd0a47bccf9c25d443958cfd29968fe4f6b22f9b07dc7e4f0a97eb729896b
+size 6651904