Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250822_175544-mg58khw0/run-mg58khw0.wandb filter=lfs diff=lfs merge=lfs -text

checkpoints/step-004000-epoch-05-loss=0.2617.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:473480ba488abbfeebbca7147215571138c18985d2a92c13bb3b5e59604e59fd
+size 4093057552

checkpoints/step-007160-epoch-10-loss=0.1436.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f801b6b80c5e6f235af6daf003b8b1a20811b6037367ccdec70f133fb56253b5
+size 4093057552

config.json

@@ -0,0 +1,58 @@
+{
+  "data_root_dir": "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/.new_maniskill_data",
+  "depth": false,
+  "global_pose": false,
+  "hf_token": ".hf_token",
+  "image_aug": false,
+  "is_grasped": false,
+  "is_resume": false,
+  "model_type": "my_vla_qwen",
+  "pretrained_checkpoint": "Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b",
+  "qpos": false,
+  "resume_epoch": null,
+  "resume_step": null,
+  "run_id": "bl_multiview_depth_set_table",
+  "run_id_note": null,
+  "run_root_dir": "myvla_exp",
+  "save_interval": 1000,
+  "seed": 7,
+  "segmentation": false,
+  "trackers": [
+    "jsonl",
+    "wandb"
+  ],
+  "vla": {
+    "action_chunk_size": 8,
+    "action_tokenizer": "extra_action_tokenizer",
+    "base_vlm": "prism-qwen25-extra-dinosiglip-224px+0_5b",
+    "compress_history": false,
+    "data_mix": "bridge",
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true,
+    "epochs": 10,
+    "expected_world_size": 8,
+    "freeze_llm_backbone": false,
+    "freeze_vision_backbone": true,
+    "global_batch_size": 512,
+    "image_sequence_len": 4,
+    "image_window_size": 1,
+    "learning_rate": 2e-05,
+    "lr_scheduler_type": "constant",
+    "max_grad_norm": 1.0,
+    "max_steps": null,
+    "per_device_batch_size": 8,
+    "reduce_in_full_precision": true,
+    "save_every_n_steps": 25000,
+    "shuffle_buffer_size": 256000,
+    "train_strategy": "fsdp-full-shard",
+    "type": "myvla-qwen-224px+mx-mshab",
+    "unfreeze_last_llm_layer": false,
+    "use_flow_matching": false,
+    "use_wrist_image": true,
+    "vla_id": "myvla-qwen-224px+mx-mshab",
+    "warmup_ratio": 0.0,
+    "weight_decay": 0.0
+  },
+  "wandb_entity": "traysen879-uc-san-diego",
+  "wandb_project": "mshab_vla"
+}

config.yaml

@@ -0,0 +1,54 @@
+data_root_dir: /home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/.new_maniskill_data
+depth: false
+global_pose: false
+hf_token: .hf_token
+image_aug: false
+is_grasped: false
+is_resume: false
+model_type: my_vla_qwen
+pretrained_checkpoint: Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b
+qpos: false
+resume_epoch: null
+resume_step: null
+run_id: bl_multiview_depth_set_table
+run_id_note: null
+run_root_dir: myvla_exp
+save_interval: 1000
+seed: 7
+segmentation: false
+trackers:
+- jsonl
+- wandb
+vla:
+  action_chunk_size: 8
+  action_tokenizer: extra_action_tokenizer
+  base_vlm: prism-qwen25-extra-dinosiglip-224px+0_5b
+  compress_history: false
+  data_mix: bridge
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+  epochs: 10
+  expected_world_size: 8
+  freeze_llm_backbone: false
+  freeze_vision_backbone: true
+  global_batch_size: 512
+  image_sequence_len: 4
+  image_window_size: 1
+  learning_rate: 2.0e-05
+  lr_scheduler_type: constant
+  max_grad_norm: 1.0
+  max_steps: null
+  per_device_batch_size: 8
+  reduce_in_full_precision: true
+  save_every_n_steps: 25000
+  shuffle_buffer_size: 256000
+  train_strategy: fsdp-full-shard
+  type: myvla-qwen-224px+mx-mshab
+  unfreeze_last_llm_layer: false
+  use_flow_matching: false
+  use_wrist_image: true
+  vla_id: myvla-qwen-224px+mx-mshab
+  warmup_ratio: 0.0
+  weight_decay: 0.0
+wandb_entity: traysen879-uc-san-diego
+wandb_project: mshab_vla

run-metrics.jsonl

@@ -0,0 +1 @@
+{"hparams": {"data_root_dir": "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/.new_maniskill_data", "depth": false, "global_pose": false, "hf_token": ".hf_token", "image_aug": false, "is_grasped": false, "is_resume": false, "model_type": "my_vla_qwen", "pretrained_checkpoint": "Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b", "qpos": false, "resume_epoch": null, "resume_step": null, "run_id": "bl_multiview_depth_set_table", "run_id_note": null, "run_root_dir": "myvla_exp", "save_interval": 1000, "seed": 7, "segmentation": false, "trackers": ["jsonl", "wandb"], "vla": {"action_chunk_size": 8, "action_tokenizer": "extra_action_tokenizer", "base_vlm": "prism-qwen25-extra-dinosiglip-224px+0_5b", "compress_history": false, "data_mix": "bridge", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "epochs": 10, "expected_world_size": 8, "freeze_llm_backbone": false, "freeze_vision_backbone": true, "global_batch_size": 512, "image_sequence_len": 4, "image_window_size": 1, "learning_rate": 2e-05, "lr_scheduler_type": "constant", "max_grad_norm": 1.0, "max_steps": null, "per_device_batch_size": 8, "reduce_in_full_precision": true, "save_every_n_steps": 25000, "shuffle_buffer_size": 256000, "train_strategy": "fsdp-full-shard", "type": "myvla-qwen-224px+mx-mshab", "unfreeze_last_llm_layer": false, "use_flow_matching": false, "use_wrist_image": true, "vla_id": "myvla-qwen-224px+mx-mshab", "warmup_ratio": 0.0, "weight_decay": 0.0}, "wandb_entity": "traysen879-uc-san-diego", "wandb_project": "mshab_vla"}, "run_id": "bl_multiview_depth_set_table"}

wandb/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2025-08-22T17:55:45.122997378Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"}
+{"time":"2025-08-22T17:55:45.417778442Z","level":"INFO","msg":"stream: created new stream","id":"mg58khw0"}
+{"time":"2025-08-22T17:55:45.417944745Z","level":"INFO","msg":"stream: started","id":"mg58khw0"}
+{"time":"2025-08-22T17:55:45.417987887Z","level":"INFO","msg":"writer: Do: started","stream_id":"mg58khw0"}
+{"time":"2025-08-22T17:55:45.418028658Z","level":"INFO","msg":"sender: started","stream_id":"mg58khw0"}
+{"time":"2025-08-22T17:55:45.418052908Z","level":"INFO","msg":"handler: started","stream_id":"mg58khw0"}
+{"time":"2025-08-22T19:05:01.607571042Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-08-23T06:27:10.330396586Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-23T06:27:10.442620306Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2025-08-23T06:27:10.450768217Z","level":"INFO","msg":"stream: closing","id":"mg58khw0"}
+{"time":"2025-08-23T06:27:10.450887019Z","level":"INFO","msg":"handler: closed","stream_id":"mg58khw0"}
+{"time":"2025-08-23T06:27:10.450965602Z","level":"INFO","msg":"sender: closed","stream_id":"mg58khw0"}
+{"time":"2025-08-23T06:27:10.45093435Z","level":"INFO","msg":"writer: Close: closed","stream_id":"mg58khw0"}
+{"time":"2025-08-23T06:27:10.45361464Z","level":"INFO","msg":"stream: closed","id":"mg58khw0"}

wandb/debug.log

@@ -0,0 +1,28 @@
+2025-08-22 17:55:44,851 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0
+2025-08-22 17:55:44,851 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Configure stats pid to 4004464
+2025-08-22 17:55:44,851 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Loading settings from /lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/wandb/settings
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:setup_run_log_directory():703] Logging user logs to myvla_exp/bl_multiview_depth_set_table/wandb/run-20250822_175544-mg58khw0/logs/debug.log
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to myvla_exp/bl_multiview_depth_set_table/wandb/run-20250822_175544-mg58khw0/logs/debug-internal.log
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:init():830] calling init triggers
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'vla': {'type': 'myvla-qwen-224px+mx-mshab', 'vla_id': 'myvla-qwen-224px+mx-mshab', 'base_vlm': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'freeze_vision_backbone': True, 'freeze_llm_backbone': False, 'unfreeze_last_llm_layer': False, 'data_mix': 'bridge', 'shuffle_buffer_size': 256000, 'epochs': 10, 'max_steps': None, 'save_every_n_steps': 25000, 'expected_world_size': 8, 'global_batch_size': 512, 'per_device_batch_size': 8, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'max_grad_norm': 1.0, 'lr_scheduler_type': 'constant', 'warmup_ratio': 0.0, 'train_strategy': 'fsdp-full-shard', 'action_tokenizer': 'extra_action_tokenizer', 'image_sequence_len': 4, 'use_wrist_image': True, 'compress_history': False, 'use_flow_matching': False, 'action_chunk_size': 8, 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'image_window_size': 1}, 'model_type': 'my_vla_qwen', 'data_root_dir': '/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/.new_maniskill_data', 'run_root_dir': 'myvla_exp', 'pretrained_checkpoint': 'Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b', 'is_resume': False, 'resume_step': None, 'resume_epoch': None, 'run_id': 'bl_multiview_depth_set_table', 'run_id_note': None, 'save_interval': 1000, 'image_aug': False, 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'mshab_vla', 'wandb_entity': 'traysen879-uc-san-diego', 'global_pose': False, 'is_grasped': False, 'qpos': False, 'depth': False, 'segmentation': False, '_wandb': {}}
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:init():871] starting backend
+2025-08-22 17:55:45,099 INFO    MainThread:4004464 [wandb_init.py:init():874] sending inform_init request
+2025-08-22 17:55:45,108 INFO    MainThread:4004464 [wandb_init.py:init():882] backend started and connected
+2025-08-22 17:55:45,114 INFO    MainThread:4004464 [wandb_init.py:init():953] updated telemetry
+2025-08-22 17:55:45,150 INFO    MainThread:4004464 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-22 17:55:45,586 INFO    MainThread:4004464 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-22 17:55:46,161 INFO    MainThread:4004464 [wandb_run.py:_console_start():2458] atexit reg
+2025-08-22 17:55:46,161 INFO    MainThread:4004464 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-08-22 17:55:46,163 INFO    MainThread:4004464 [wandb_run.py:_redirect():2375] Wrapping output streams.
+2025-08-22 17:55:46,163 INFO    MainThread:4004464 [wandb_run.py:_redirect():2398] Redirects installed.
+2025-08-22 17:55:46,173 INFO    MainThread:4004464 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-23 06:27:09,979 INFO    MainThread:4004464 [wandb_run.py:_finish():2224] finishing run traysen879-uc-san-diego/mshab_vla/mg58khw0
+2025-08-23 06:27:09,980 INFO    MainThread:4004464 [wandb_run.py:_atexit_cleanup():2423] got exitcode: 0
+2025-08-23 06:27:09,980 INFO    MainThread:4004464 [wandb_run.py:_restore():2405] restore
+2025-08-23 06:27:09,981 INFO    MainThread:4004464 [wandb_run.py:_restore():2411] restore done
+2025-08-23 06:27:10,444 INFO    MainThread:4004464 [wandb_run.py:_footer_history_summary_info():3903] rendering history
+2025-08-23 06:27:10,446 INFO    MainThread:4004464 [wandb_run.py:_footer_history_summary_info():3935] rendering summary
+2025-08-23 06:27:10,446 INFO    MainThread:4004464 [wandb_run.py:_footer_sync_info():3864] logging synced files

wandb/run-20250822_172424-grd0n90q/files/config.yaml

@@ -0,0 +1,210 @@
+_wandb:
+    value:
+        cli_version: 0.21.0
+        e:
+            dnq1lm07509oy3nh24mhk899wrek9b94:
+                args:
+                    - --pretrained_checkpoint
+                    - Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b
+                    - --model_type
+                    - my_vla_qwen
+                    - --vla.type
+                    - myvla-qwen-224px+mx-mshab
+                    - --vla.expected_world_size
+                    - "8"
+                    - --vla.global_batch_size
+                    - "512"
+                    - --vla.per_device_batch_size
+                    - "8"
+                    - --vla.learning_rate
+                    - "2e-5"
+                    - --vla.freeze_vision_backbone
+                    - "True"
+                    - --vla.freeze_llm_backbone
+                    - "False"
+                    - --vla.use_flow_matching
+                    - "False"
+                    - --vla.compress_history
+                    - "False"
+                    - --vla.image_sequence_len
+                    - "4"
+                    - --vla.image_window_size
+                    - "1"
+                    - --vla.epochs
+                    - "10"
+                    - --save_interval
+                    - "1000"
+                    - --run_id
+                    - bl_multiview_depth_set_table
+                codePath: vla-scripts/train.py
+                codePathLocal: vla-scripts/train.py
+                cpu_count: 240
+                cpu_count_logical: 240
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "20812690710528"
+                        used: "36965724160"
+                email: traysen879@gmail.com
+                executable: /home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/bin/python3.10
+                git:
+                    commit: 409e4c9a165115624c271028e9b3ee335991b747
+                    remote: https://github.com/TRS07170/myvla.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-47bfdc91-9dec-cf54-0e0a-aa57ab6fb106
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-55a7184b-b6dc-a8b3-67d5-a65679215c83
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1de758e0-e4a9-e2e9-027c-17f65db8a69e
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-d7f94efd-7e10-156f-fe37-e505ae7b62b1
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-813530b2-64f0-5fa3-3568-3811977d3b92
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-7eac47dc-0da1-f6b2-d261-8ab3a5d4ed03
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-335150e5-634c-68e2-4930-656c95e62244
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-b3ee08d0-187c-8f80-06d5-c46759764c41
+                host: 164-152-109-69
+                memory:
+                    total: "1902324936704"
+                os: Linux-6.8.0-60-generic-x86_64-with-glibc2.35
+                program: /lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py
+                python: CPython 3.10.18
+                root: myvla_exp/bl_multiview_depth_set_table
+                startedAt: "2025-08-22T17:24:24.381806Z"
+                writerId: dnq1lm07509oy3nh24mhk899wrek9b94
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 63
+                - 71
+            "2":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 63
+                - 71
+            "3":
+                - 13
+                - 16
+                - 61
+            "4": 3.10.18
+            "5": 0.21.0
+            "6": 4.40.1
+            "12": 0.21.0
+            "13": linux-x86_64
+data_root_dir:
+    value: /home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/.new_maniskill_data
+depth:
+    value: false
+global_pose:
+    value: false
+hf_token:
+    value: .hf_token
+image_aug:
+    value: false
+is_grasped:
+    value: false
+is_resume:
+    value: false
+model_type:
+    value: my_vla_qwen
+pretrained_checkpoint:
+    value: Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b
+qpos:
+    value: false
+resume_epoch:
+    value: null
+resume_step:
+    value: null
+run_id:
+    value: bl_multiview_depth_set_table
+run_id_note:
+    value: null
+run_root_dir:
+    value: myvla_exp
+save_interval:
+    value: 1000
+seed:
+    value: 7
+segmentation:
+    value: false
+trackers:
+    value:
+        - jsonl
+        - wandb
+vla:
+    value:
+        action_chunk_size: 8
+        action_tokenizer: extra_action_tokenizer
+        base_vlm: prism-qwen25-extra-dinosiglip-224px+0_5b
+        compress_history: false
+        data_mix: bridge
+        enable_gradient_checkpointing: true
+        enable_mixed_precision_training: true
+        epochs: 10
+        expected_world_size: 8
+        freeze_llm_backbone: false
+        freeze_vision_backbone: true
+        global_batch_size: 512
+        image_sequence_len: 4
+        image_window_size: 1
+        learning_rate: 2e-05
+        lr_scheduler_type: constant
+        max_grad_norm: 1
+        max_steps: null
+        per_device_batch_size: 8
+        reduce_in_full_precision: true
+        save_every_n_steps: 25000
+        shuffle_buffer_size: 256000
+        train_strategy: fsdp-full-shard
+        type: myvla-qwen-224px+mx-mshab
+        unfreeze_last_llm_layer: false
+        use_flow_matching: false
+        use_wrist_image: true
+        vla_id: myvla-qwen-224px+mx-mshab
+        warmup_ratio: 0
+        weight_decay: 0
+wandb_entity:
+    value: traysen879-uc-san-diego
+wandb_project:
+    value: mshab_vla

wandb/run-20250822_172424-grd0n90q/files/output.log

@@ -0,0 +1,75 @@
+08/22 [17:24:25] INFO     | >> [*] Starting VLA Training Loop                                                                                                                     ]8;id=817857;file:///lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py\train.py]8;;\:]8;id=998125;file:///lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py#322\322]8;;\
+Traceback (most recent call last):                                                                                                                                                            
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py", line 342, in <module>
+    train()
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/draccus/argparsing.py", line 203, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py", line 323, in train
+    train_strategy.run_vla_training(
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/prismatic/training/strategies/base_strategy.py", line 342, in run_vla_training
+    output, aux_loss = self.vlm(
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/prismatic/models/vlas/myvla.py", line 277, in forward
+    output = super().forward(
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/prismatic/models/vlms/prismatic.py", line 373, in forward
+    patch_features = self.vision_backbone({k: pixel_values[k][multimodal_indices] for k in pixel_values})
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/prismatic/models/backbones/vision/dinosiglip_vit.py", line 169, in forward
+    patches = compute_sequence_patches(pixel_values, featurizers, self.image_sequence_len)
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/prismatic/models/backbones/vision/base_vision.py", line 47, in compute_sequence_patches
+    patches[k] = merge_two_dims(sequence_combine_call_split(trunc_pixels_k, featurizers[k]), start_dim=1)
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/prismatic/util/torch_utils.py", line 106, in sequence_combine_call_split
+    flat_outputs = fn(flat_sequence)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/prismatic/models/backbones/vision/base_vision.py", line 31, in wrapper
+    result = fn(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/timm/models/vision_transformer.py", line 644, in get_intermediate_layers
+    outputs = self._intermediate_layers(x, n)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/timm/models/vision_transformer.py", line 626, in _intermediate_layers
+    x = blk(x)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/timm/models/vision_transformer.py", line 157, in forward
+    x = x + self.drop_path2(self.ls2(self.mlp(self.norm2(x))))
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/timm/layers/mlp.py", line 46, in forward
+    x = self.fc2(x)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/lib/python3.10/site-packages/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+KeyboardInterrupt

wandb/run-20250822_172424-grd0n90q/files/requirements.txt

@@ -0,0 +1,144 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.1
+sentencepiece==0.1.99
+mpmath==1.3.0
+libclang==18.1.1
+flatbuffers==25.2.10
+zipp==3.23.0
+wrapt==1.14.1
+urllib3==2.5.0
+typing_extensions==4.14.1
+typeguard==2.13.3
+tqdm==4.67.1
+toml==0.10.2
+termcolor==3.1.0
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-estimator==2.15.0
+tensorboard-data-server==0.7.2
+sympy==1.14.0
+smmap==5.0.2
+six==1.17.0
+safetensors==0.5.3
+regex==2025.7.34
+PyYAML==6.0.2
+pyparsing==3.2.3
+Pygments==2.19.2
+pyasn1==0.6.1
+psutil==7.0.0
+protobuf==4.21.12
+platformdirs==4.3.8
+pillow==11.3.0
+packaging==25.0
+opt_einsum==3.4.0
+oauthlib==3.3.1
+nvidia-nvtx-cu12==12.1.105
+nvidia-nvjitlink-cu12==12.9.86
+nvidia-nccl-cu12==2.19.3
+nvidia-curand-cu12==10.3.2.106
+nvidia-cufft-cu12==11.0.2.54
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cublas-cu12==12.1.3.1
+numpy==1.26.4
+networkx==3.4.2
+mypy_extensions==1.1.0
+mergedeep==1.3.4
+mdurl==0.1.2
+MarkupSafe==3.0.2
+Markdown==3.8.2
+kiwisolver==1.4.8
+keras==2.15.0
+importlib_resources==6.5.2
+idna==3.10
+hf-xet==1.1.5
+grpcio==1.74.0
+gast==0.6.0
+fsspec==2025.7.0
+fonttools==4.59.0
+filelock==3.18.0
+etils==1.13.0
+einops==0.8.1
+cycler==0.12.1
+click==8.2.1
+charset-normalizer==3.4.2
+certifi==2025.8.3
+cachetools==5.5.2
+attrs==25.3.0
+annotated-types==0.7.0
+absl-py==2.3.1
+Werkzeug==3.1.3
+typing-inspection==0.4.1
+typing-inspect==0.9.0
+triton==2.2.0
+trimesh==4.7.1
+tensorflow-metadata==1.17.2
+tensorflow-addons==0.23.0
+sentry-sdk==2.34.1
+scipy==1.15.3
+rsa==4.9.1
+requests==2.32.4
+pyyaml-include==1.4.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.33.2
+pyasn1_modules==0.4.2
+promise==2.3
+OpenEXR==3.3.5
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-cudnn-cu12==8.9.2.26
+ml-dtypes==0.2.0
+markdown-it-py==3.0.0
+jsonlines==4.0.0
+json-numpy==2.1.1
+Jinja2==3.1.6
+h5py==3.14.0
+google-pasta==0.2.0
+gitdb==4.0.12
+dm-tree==0.1.9
+contourpy==1.3.2
+astunparse==1.6.3
+rich==14.1.0
+requests-oauthlib==2.0.0
+pydantic==2.11.7
+nvidia-cusolver-cu12==11.4.5.107
+matplotlib==3.10.5
+huggingface-hub==0.34.3
+google-auth==2.40.3
+GitPython==3.1.45
+draccus==0.8.0
+wandb==0.21.0
+torch==2.2.0
+tokenizers==0.19.1
+google-auth-oauthlib==1.2.2
+array_record==0.7.2
+transformers==4.40.1
+torchvision==0.17.0
+torchaudio==2.2.0
+tensorboard==2.15.2
+accelerate==1.9.0
+timm==0.9.10
+tensorflow-datasets==4.9.3
+tensorflow==2.15.0
+peft==0.11.1
+tensorflow-graphics==2021.12.3
+dlimp==0.0.1
+openvla==0.0.3
+ninja==1.11.1.4
+flash-attn==2.5.5
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

wandb/run-20250822_172424-grd0n90q/files/wandb-metadata.json

@@ -0,0 +1,123 @@
+{
+  "os":  "Linux-6.8.0-60-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2025-08-22T17:24:24.381806Z",
+  "args":  [
+    "--pretrained_checkpoint",
+    "Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b",
+    "--model_type",
+    "my_vla_qwen",
+    "--vla.type",
+    "myvla-qwen-224px+mx-mshab",
+    "--vla.expected_world_size",
+    "8",
+    "--vla.global_batch_size",
+    "512",
+    "--vla.per_device_batch_size",
+    "8",
+    "--vla.learning_rate",
+    "2e-5",
+    "--vla.freeze_vision_backbone",
+    "True",
+    "--vla.freeze_llm_backbone",
+    "False",
+    "--vla.use_flow_matching",
+    "False",
+    "--vla.compress_history",
+    "False",
+    "--vla.image_sequence_len",
+    "4",
+    "--vla.image_window_size",
+    "1",
+    "--vla.epochs",
+    "10",
+    "--save_interval",
+    "1000",
+    "--run_id",
+    "bl_multiview_depth_set_table"
+  ],
+  "program":  "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py",
+  "codePath":  "vla-scripts/train.py",
+  "codePathLocal":  "vla-scripts/train.py",
+  "git":  {
+    "remote":  "https://github.com/TRS07170/myvla.git",
+    "commit":  "409e4c9a165115624c271028e9b3ee335991b747"
+  },
+  "email":  "traysen879@gmail.com",
+  "root":  "myvla_exp/bl_multiview_depth_set_table",
+  "host":  "164-152-109-69",
+  "executable":  "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/bin/python3.10",
+  "cpu_count":  240,
+  "cpu_count_logical":  240,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "20812690710528",
+      "used":  "36965724160"
+    }
+  },
+  "memory":  {
+    "total":  "1902324936704"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-47bfdc91-9dec-cf54-0e0a-aa57ab6fb106"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-55a7184b-b6dc-a8b3-67d5-a65679215c83"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-1de758e0-e4a9-e2e9-027c-17f65db8a69e"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-d7f94efd-7e10-156f-fe37-e505ae7b62b1"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-813530b2-64f0-5fa3-3568-3811977d3b92"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-7eac47dc-0da1-f6b2-d261-8ab3a5d4ed03"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-335150e5-634c-68e2-4930-656c95e62244"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-b3ee08d0-187c-8f80-06d5-c46759764c41"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "dnq1lm07509oy3nh24mhk899wrek9b94"
+}

wandb/run-20250822_172424-grd0n90q/files/wandb-summary.json

@@ -0,0 +1 @@
+{"VLA Train/L1 Loss":0.6152674662097849,"VLA Train/Step Time":9.066001892089844,"_step":18,"_timestamp":1.7558836377797978e+09,"VLA Train/Step":18,"VLA Train/Loss (Raw)":0.5707007646560669,"_runtime":179.53910675,"VLA Train/Auxiliary Loss":0,"VLA Train/Action Token Accuracy":0.28155338764190674,"VLA Train/Epoch":0,"VLA Train/Loss":0.5707007646560669,"_wandb":{"runtime":179},"VLA Train/Learning Rate":2e-05}

wandb/run-20250822_172424-grd0n90q/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-08-22T17:24:24.667654596Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_342id0h/port-4002856.txt","pid":4002856,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-08-22T17:24:24.66880959Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4002856-4003306-1128414806/socket","Net":"unix"}}
+{"time":"2025-08-22T17:24:24.668943304Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4002856}
+{"time":"2025-08-22T17:24:24.688040644Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-08-22T17:24:24.702187716Z","level":"INFO","msg":"handleInformInit: received","streamId":"grd0n90q","id":"1(@)"}
+{"time":"2025-08-22T17:24:24.994195243Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"grd0n90q","id":"1(@)"}
+{"time":"2025-08-22T17:27:24.702921547Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-08-22T17:27:24.703114991Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-08-22T17:27:24.703256274Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4002856-4003306-1128414806/socket","Net":"unix"}}
+{"time":"2025-08-22T17:27:24.70309939Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-08-22T17:27:24.703325336Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-08-22T17:27:24.841847315Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-08-22T17:27:24.841900846Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-08-22T17:27:24.841914286Z","level":"INFO","msg":"server is closed"}

wandb/run-20250822_172424-grd0n90q/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2025-08-22T17:24:24.706169983Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"}
+{"time":"2025-08-22T17:24:24.993938237Z","level":"INFO","msg":"stream: created new stream","id":"grd0n90q"}
+{"time":"2025-08-22T17:24:24.99408303Z","level":"INFO","msg":"stream: started","id":"grd0n90q"}
+{"time":"2025-08-22T17:24:24.994179082Z","level":"INFO","msg":"handler: started","stream_id":"grd0n90q"}
+{"time":"2025-08-22T17:24:24.994227393Z","level":"INFO","msg":"sender: started","stream_id":"grd0n90q"}
+{"time":"2025-08-22T17:24:24.994136561Z","level":"INFO","msg":"writer: Do: started","stream_id":"grd0n90q"}
+{"time":"2025-08-22T17:27:24.703158922Z","level":"INFO","msg":"stream: closing","id":"grd0n90q"}
+{"time":"2025-08-22T17:27:24.785199848Z","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2025-08-22T17:27:24.785511865Z","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404: {\"data\":{\"createRunFiles\":null},\"errors\":[{\"message\":\"run mshab_vla/grd0n90q not found during createRunFiles\",\"path\":[\"createRunFiles\"]}]}"}
+{"time":"2025-08-22T17:27:24.790789631Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-22T17:27:24.831280893Z","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/files/traysen879-uc-san-diego/mshab_vla/grd0n90q/file_stream"}
+{"time":"2025-08-22T17:27:24.831581099Z","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 404 Not Found path=files/traysen879-uc-san-diego/mshab_vla/grd0n90q/file_stream: {\"error\":\"run mshab_vla/grd0n90q not found while streaming file\"}"}
+{"time":"2025-08-22T17:27:24.834877722Z","level":"INFO","msg":"handler: closed","stream_id":"grd0n90q"}
+{"time":"2025-08-22T17:27:24.834904482Z","level":"INFO","msg":"writer: Close: closed","stream_id":"grd0n90q"}
+{"time":"2025-08-22T17:27:24.834946803Z","level":"INFO","msg":"sender: closed","stream_id":"grd0n90q"}
+{"time":"2025-08-22T17:27:24.839882542Z","level":"INFO","msg":"stream: closed","id":"grd0n90q"}

wandb/run-20250822_172424-grd0n90q/logs/debug.log

@@ -0,0 +1,22 @@
+2025-08-22 17:24:24,427 INFO    MainThread:4002856 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0
+2025-08-22 17:24:24,427 INFO    MainThread:4002856 [wandb_setup.py:_flush():80] Configure stats pid to 4002856
+2025-08-22 17:24:24,427 INFO    MainThread:4002856 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-08-22 17:24:24,430 INFO    MainThread:4002856 [wandb_setup.py:_flush():80] Loading settings from /lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/wandb/settings
+2025-08-22 17:24:24,432 INFO    MainThread:4002856 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-22 17:24:24,432 INFO    MainThread:4002856 [wandb_init.py:setup_run_log_directory():703] Logging user logs to myvla_exp/bl_multiview_depth_set_table/wandb/run-20250822_172424-grd0n90q/logs/debug.log
+2025-08-22 17:24:24,434 INFO    MainThread:4002856 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to myvla_exp/bl_multiview_depth_set_table/wandb/run-20250822_172424-grd0n90q/logs/debug-internal.log
+2025-08-22 17:24:24,436 INFO    MainThread:4002856 [wandb_init.py:init():830] calling init triggers
+2025-08-22 17:24:24,438 INFO    MainThread:4002856 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'vla': {'type': 'myvla-qwen-224px+mx-mshab', 'vla_id': 'myvla-qwen-224px+mx-mshab', 'base_vlm': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'freeze_vision_backbone': True, 'freeze_llm_backbone': False, 'unfreeze_last_llm_layer': False, 'data_mix': 'bridge', 'shuffle_buffer_size': 256000, 'epochs': 10, 'max_steps': None, 'save_every_n_steps': 25000, 'expected_world_size': 8, 'global_batch_size': 512, 'per_device_batch_size': 8, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'max_grad_norm': 1.0, 'lr_scheduler_type': 'constant', 'warmup_ratio': 0.0, 'train_strategy': 'fsdp-full-shard', 'action_tokenizer': 'extra_action_tokenizer', 'image_sequence_len': 4, 'use_wrist_image': True, 'compress_history': False, 'use_flow_matching': False, 'action_chunk_size': 8, 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'image_window_size': 1}, 'model_type': 'my_vla_qwen', 'data_root_dir': '/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/.new_maniskill_data', 'run_root_dir': 'myvla_exp', 'pretrained_checkpoint': 'Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b', 'is_resume': False, 'resume_step': None, 'resume_epoch': None, 'run_id': 'bl_multiview_depth_set_table', 'run_id_note': None, 'save_interval': 1000, 'image_aug': False, 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'mshab_vla', 'wandb_entity': 'traysen879-uc-san-diego', 'global_pose': False, 'is_grasped': False, 'qpos': False, 'depth': False, 'segmentation': False, '_wandb': {}}
+2025-08-22 17:24:24,438 INFO    MainThread:4002856 [wandb_init.py:init():871] starting backend
+2025-08-22 17:24:24,688 INFO    MainThread:4002856 [wandb_init.py:init():874] sending inform_init request
+2025-08-22 17:24:24,698 INFO    MainThread:4002856 [wandb_init.py:init():882] backend started and connected
+2025-08-22 17:24:24,703 INFO    MainThread:4002856 [wandb_init.py:init():953] updated telemetry
+2025-08-22 17:24:24,733 INFO    MainThread:4002856 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-22 17:24:25,161 INFO    MainThread:4002856 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-22 17:24:25,682 INFO    MainThread:4002856 [wandb_run.py:_console_start():2458] atexit reg
+2025-08-22 17:24:25,683 INFO    MainThread:4002856 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-08-22 17:24:25,685 INFO    MainThread:4002856 [wandb_run.py:_redirect():2375] Wrapping output streams.
+2025-08-22 17:24:25,685 INFO    MainThread:4002856 [wandb_run.py:_redirect():2398] Redirects installed.
+2025-08-22 17:24:25,696 INFO    MainThread:4002856 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-22 17:27:24,701 INFO    MsgRouterThr:4002856 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

wandb/run-20250822_175544-mg58khw0/files/config.yaml

@@ -0,0 +1,211 @@
+_wandb:
+    value:
+        cli_version: 0.21.0
+        e:
+            kigmkgl7ewlmvaaruo3ggp33h5rhvks0:
+                args:
+                    - --pretrained_checkpoint
+                    - Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b
+                    - --model_type
+                    - my_vla_qwen
+                    - --vla.type
+                    - myvla-qwen-224px+mx-mshab
+                    - --vla.expected_world_size
+                    - "8"
+                    - --vla.global_batch_size
+                    - "512"
+                    - --vla.per_device_batch_size
+                    - "8"
+                    - --vla.learning_rate
+                    - "2e-5"
+                    - --vla.freeze_vision_backbone
+                    - "True"
+                    - --vla.freeze_llm_backbone
+                    - "False"
+                    - --vla.use_flow_matching
+                    - "False"
+                    - --vla.compress_history
+                    - "False"
+                    - --vla.image_sequence_len
+                    - "4"
+                    - --vla.image_window_size
+                    - "1"
+                    - --vla.epochs
+                    - "10"
+                    - --save_interval
+                    - "1000"
+                    - --run_id
+                    - bl_multiview_depth_set_table
+                codePath: vla-scripts/train.py
+                codePathLocal: vla-scripts/train.py
+                cpu_count: 240
+                cpu_count_logical: 240
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "20812690710528"
+                        used: "36965580800"
+                email: traysen879@gmail.com
+                executable: /home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/bin/python3.10
+                git:
+                    commit: 409e4c9a165115624c271028e9b3ee335991b747
+                    remote: https://github.com/TRS07170/myvla.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-47bfdc91-9dec-cf54-0e0a-aa57ab6fb106
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-55a7184b-b6dc-a8b3-67d5-a65679215c83
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1de758e0-e4a9-e2e9-027c-17f65db8a69e
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-d7f94efd-7e10-156f-fe37-e505ae7b62b1
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-813530b2-64f0-5fa3-3568-3811977d3b92
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-7eac47dc-0da1-f6b2-d261-8ab3a5d4ed03
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-335150e5-634c-68e2-4930-656c95e62244
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-b3ee08d0-187c-8f80-06d5-c46759764c41
+                host: 164-152-109-69
+                memory:
+                    total: "1902324936704"
+                os: Linux-6.8.0-60-generic-x86_64-with-glibc2.35
+                program: /lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py
+                python: CPython 3.10.18
+                root: myvla_exp/bl_multiview_depth_set_table
+                startedAt: "2025-08-22T17:55:44.807018Z"
+                writerId: kigmkgl7ewlmvaaruo3ggp33h5rhvks0
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 63
+                - 71
+            "2":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 63
+                - 71
+            "3":
+                - 2
+                - 13
+                - 16
+                - 61
+            "4": 3.10.18
+            "5": 0.21.0
+            "6": 4.40.1
+            "12": 0.21.0
+            "13": linux-x86_64
+data_root_dir:
+    value: /home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/.new_maniskill_data
+depth:
+    value: false
+global_pose:
+    value: false
+hf_token:
+    value: .hf_token
+image_aug:
+    value: false
+is_grasped:
+    value: false
+is_resume:
+    value: false
+model_type:
+    value: my_vla_qwen
+pretrained_checkpoint:
+    value: Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b
+qpos:
+    value: false
+resume_epoch:
+    value: null
+resume_step:
+    value: null
+run_id:
+    value: bl_multiview_depth_set_table
+run_id_note:
+    value: null
+run_root_dir:
+    value: myvla_exp
+save_interval:
+    value: 1000
+seed:
+    value: 7
+segmentation:
+    value: false
+trackers:
+    value:
+        - jsonl
+        - wandb
+vla:
+    value:
+        action_chunk_size: 8
+        action_tokenizer: extra_action_tokenizer
+        base_vlm: prism-qwen25-extra-dinosiglip-224px+0_5b
+        compress_history: false
+        data_mix: bridge
+        enable_gradient_checkpointing: true
+        enable_mixed_precision_training: true
+        epochs: 10
+        expected_world_size: 8
+        freeze_llm_backbone: false
+        freeze_vision_backbone: true
+        global_batch_size: 512
+        image_sequence_len: 4
+        image_window_size: 1
+        learning_rate: 2e-05
+        lr_scheduler_type: constant
+        max_grad_norm: 1
+        max_steps: null
+        per_device_batch_size: 8
+        reduce_in_full_precision: true
+        save_every_n_steps: 25000
+        shuffle_buffer_size: 256000
+        train_strategy: fsdp-full-shard
+        type: myvla-qwen-224px+mx-mshab
+        unfreeze_last_llm_layer: false
+        use_flow_matching: false
+        use_wrist_image: true
+        vla_id: myvla-qwen-224px+mx-mshab
+        warmup_ratio: 0
+        weight_decay: 0
+wandb_entity:
+    value: traysen879-uc-san-diego
+wandb_project:
+    value: mshab_vla

wandb/run-20250822_175544-mg58khw0/files/output.log

@@ -0,0 +1,3 @@
+08/22 [17:55:46] INFO     | >> [*] Starting VLA Training Loop                                                                                                                     ]8;id=817857;file:///lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py\train.py]8;;\:]8;id=998125;file:///lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py#322\322]8;;\
+                                                                                                                                                                                              
+08/23 [06:27:09] INFO     | >> [*] Done with Training =>> Finalizing Metrics                                                                                                      ]8;id=454536;file:///lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py\train.py]8;;\:]8;id=267836;file:///lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py#332\332]8;;\

wandb/run-20250822_175544-mg58khw0/files/requirements.txt

@@ -0,0 +1,144 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.1
+sentencepiece==0.1.99
+mpmath==1.3.0
+libclang==18.1.1
+flatbuffers==25.2.10
+zipp==3.23.0
+wrapt==1.14.1
+urllib3==2.5.0
+typing_extensions==4.14.1
+typeguard==2.13.3
+tqdm==4.67.1
+toml==0.10.2
+termcolor==3.1.0
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-estimator==2.15.0
+tensorboard-data-server==0.7.2
+sympy==1.14.0
+smmap==5.0.2
+six==1.17.0
+safetensors==0.5.3
+regex==2025.7.34
+PyYAML==6.0.2
+pyparsing==3.2.3
+Pygments==2.19.2
+pyasn1==0.6.1
+psutil==7.0.0
+protobuf==4.21.12
+platformdirs==4.3.8
+pillow==11.3.0
+packaging==25.0
+opt_einsum==3.4.0
+oauthlib==3.3.1
+nvidia-nvtx-cu12==12.1.105
+nvidia-nvjitlink-cu12==12.9.86
+nvidia-nccl-cu12==2.19.3
+nvidia-curand-cu12==10.3.2.106
+nvidia-cufft-cu12==11.0.2.54
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cublas-cu12==12.1.3.1
+numpy==1.26.4
+networkx==3.4.2
+mypy_extensions==1.1.0
+mergedeep==1.3.4
+mdurl==0.1.2
+MarkupSafe==3.0.2
+Markdown==3.8.2
+kiwisolver==1.4.8
+keras==2.15.0
+importlib_resources==6.5.2
+idna==3.10
+hf-xet==1.1.5
+grpcio==1.74.0
+gast==0.6.0
+fsspec==2025.7.0
+fonttools==4.59.0
+filelock==3.18.0
+etils==1.13.0
+einops==0.8.1
+cycler==0.12.1
+click==8.2.1
+charset-normalizer==3.4.2
+certifi==2025.8.3
+cachetools==5.5.2
+attrs==25.3.0
+annotated-types==0.7.0
+absl-py==2.3.1
+Werkzeug==3.1.3
+typing-inspection==0.4.1
+typing-inspect==0.9.0
+triton==2.2.0
+trimesh==4.7.1
+tensorflow-metadata==1.17.2
+tensorflow-addons==0.23.0
+sentry-sdk==2.34.1
+scipy==1.15.3
+rsa==4.9.1
+requests==2.32.4
+pyyaml-include==1.4.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.33.2
+pyasn1_modules==0.4.2
+promise==2.3
+OpenEXR==3.3.5
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-cudnn-cu12==8.9.2.26
+ml-dtypes==0.2.0
+markdown-it-py==3.0.0
+jsonlines==4.0.0
+json-numpy==2.1.1
+Jinja2==3.1.6
+h5py==3.14.0
+google-pasta==0.2.0
+gitdb==4.0.12
+dm-tree==0.1.9
+contourpy==1.3.2
+astunparse==1.6.3
+rich==14.1.0
+requests-oauthlib==2.0.0
+pydantic==2.11.7
+nvidia-cusolver-cu12==11.4.5.107
+matplotlib==3.10.5
+huggingface-hub==0.34.3
+google-auth==2.40.3
+GitPython==3.1.45
+draccus==0.8.0
+wandb==0.21.0
+torch==2.2.0
+tokenizers==0.19.1
+google-auth-oauthlib==1.2.2
+array_record==0.7.2
+transformers==4.40.1
+torchvision==0.17.0
+torchaudio==2.2.0
+tensorboard==2.15.2
+accelerate==1.9.0
+timm==0.9.10
+tensorflow-datasets==4.9.3
+tensorflow==2.15.0
+peft==0.11.1
+tensorflow-graphics==2021.12.3
+dlimp==0.0.1
+openvla==0.0.3
+ninja==1.11.1.4
+flash-attn==2.5.5
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

wandb/run-20250822_175544-mg58khw0/files/wandb-metadata.json

@@ -0,0 +1,123 @@
+{
+  "os":  "Linux-6.8.0-60-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2025-08-22T17:55:44.807018Z",
+  "args":  [
+    "--pretrained_checkpoint",
+    "Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b",
+    "--model_type",
+    "my_vla_qwen",
+    "--vla.type",
+    "myvla-qwen-224px+mx-mshab",
+    "--vla.expected_world_size",
+    "8",
+    "--vla.global_batch_size",
+    "512",
+    "--vla.per_device_batch_size",
+    "8",
+    "--vla.learning_rate",
+    "2e-5",
+    "--vla.freeze_vision_backbone",
+    "True",
+    "--vla.freeze_llm_backbone",
+    "False",
+    "--vla.use_flow_matching",
+    "False",
+    "--vla.compress_history",
+    "False",
+    "--vla.image_sequence_len",
+    "4",
+    "--vla.image_window_size",
+    "1",
+    "--vla.epochs",
+    "10",
+    "--save_interval",
+    "1000",
+    "--run_id",
+    "bl_multiview_depth_set_table"
+  ],
+  "program":  "/lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/vla-scripts/train.py",
+  "codePath":  "vla-scripts/train.py",
+  "codePathLocal":  "vla-scripts/train.py",
+  "git":  {
+    "remote":  "https://github.com/TRS07170/myvla.git",
+    "commit":  "409e4c9a165115624c271028e9b3ee335991b747"
+  },
+  "email":  "traysen879@gmail.com",
+  "root":  "myvla_exp/bl_multiview_depth_set_table",
+  "host":  "164-152-109-69",
+  "executable":  "/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/miniconda3/envs/myvla/bin/python3.10",
+  "cpu_count":  240,
+  "cpu_count_logical":  240,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "20812690710528",
+      "used":  "36965580800"
+    }
+  },
+  "memory":  {
+    "total":  "1902324936704"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-47bfdc91-9dec-cf54-0e0a-aa57ab6fb106"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-55a7184b-b6dc-a8b3-67d5-a65679215c83"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-1de758e0-e4a9-e2e9-027c-17f65db8a69e"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-d7f94efd-7e10-156f-fe37-e505ae7b62b1"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-813530b2-64f0-5fa3-3568-3811977d3b92"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-7eac47dc-0da1-f6b2-d261-8ab3a5d4ed03"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-335150e5-634c-68e2-4930-656c95e62244"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-b3ee08d0-187c-8f80-06d5-c46759764c41"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "kigmkgl7ewlmvaaruo3ggp33h5rhvks0"
+}

wandb/run-20250822_175544-mg58khw0/files/wandb-summary.json

@@ -0,0 +1 @@
+{"VLA Train/Loss (Raw)":0.1435861736536026,"VLA Train/Step Time":6.651643753051758,"_step":7160,"_timestamp":1.7559304174719603e+09,"_wandb":{"runtime":45084},"_runtime":45084.391904516,"VLA Train/Loss":0.1435861736536026,"VLA Train/Auxiliary Loss":0,"VLA Train/Action Token Accuracy":0.6699029207229614,"VLA Train/Step":7160,"VLA Train/Epoch":10,"VLA Train/Learning Rate":2e-05,"VLA Train/L1 Loss":0.07942128307633732}

wandb/run-20250822_175544-mg58khw0/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2025-08-22T17:55:44.993792062Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpd1k24cho/port-4004464.txt","pid":4004464,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-08-22T17:55:44.995074839Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4004464}
+{"time":"2025-08-22T17:55:44.995034278Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4004464-4005403-1358423967/socket","Net":"unix"}}
+{"time":"2025-08-22T17:55:45.098967428Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-08-22T17:55:45.118162901Z","level":"INFO","msg":"handleInformInit: received","streamId":"mg58khw0","id":"1(@)"}
+{"time":"2025-08-22T17:55:45.417996836Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"mg58khw0","id":"1(@)"}
+{"time":"2025-08-23T06:27:10.450725106Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"mg58khw0","id":"1(@)"}
+{"time":"2025-08-23T06:27:10.45584368Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"mg58khw0","id":"1(@)"}
+{"time":"2025-08-23T06:30:42.448001203Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-08-23T06:30:42.448106914Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-08-23T06:30:42.448117954Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-08-23T06:30:42.448183167Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-08-23T06:30:42.44832581Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-08-23T06:30:42.448204876Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4004464-4005403-1358423967/socket","Net":"unix"}}
+{"time":"2025-08-23T06:30:42.44833905Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-08-23T06:30:42.448423102Z","level":"INFO","msg":"server is closed"}

wandb/run-20250822_175544-mg58khw0/logs/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2025-08-22T17:55:45.122997378Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"}
+{"time":"2025-08-22T17:55:45.417778442Z","level":"INFO","msg":"stream: created new stream","id":"mg58khw0"}
+{"time":"2025-08-22T17:55:45.417944745Z","level":"INFO","msg":"stream: started","id":"mg58khw0"}
+{"time":"2025-08-22T17:55:45.417987887Z","level":"INFO","msg":"writer: Do: started","stream_id":"mg58khw0"}
+{"time":"2025-08-22T17:55:45.418028658Z","level":"INFO","msg":"sender: started","stream_id":"mg58khw0"}
+{"time":"2025-08-22T17:55:45.418052908Z","level":"INFO","msg":"handler: started","stream_id":"mg58khw0"}
+{"time":"2025-08-22T19:05:01.607571042Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-08-23T06:27:10.330396586Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-23T06:27:10.442620306Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2025-08-23T06:27:10.450768217Z","level":"INFO","msg":"stream: closing","id":"mg58khw0"}
+{"time":"2025-08-23T06:27:10.450887019Z","level":"INFO","msg":"handler: closed","stream_id":"mg58khw0"}
+{"time":"2025-08-23T06:27:10.450965602Z","level":"INFO","msg":"sender: closed","stream_id":"mg58khw0"}
+{"time":"2025-08-23T06:27:10.45093435Z","level":"INFO","msg":"writer: Close: closed","stream_id":"mg58khw0"}
+{"time":"2025-08-23T06:27:10.45361464Z","level":"INFO","msg":"stream: closed","id":"mg58khw0"}

wandb/run-20250822_175544-mg58khw0/logs/debug.log

@@ -0,0 +1,28 @@
+2025-08-22 17:55:44,851 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0
+2025-08-22 17:55:44,851 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Configure stats pid to 4004464
+2025-08-22 17:55:44,851 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Loading settings from /lambda/nfs/jianwen-us-midwest-1/tulab/ruisen/myvla/wandb/settings
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:setup_run_log_directory():703] Logging user logs to myvla_exp/bl_multiview_depth_set_table/wandb/run-20250822_175544-mg58khw0/logs/debug.log
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to myvla_exp/bl_multiview_depth_set_table/wandb/run-20250822_175544-mg58khw0/logs/debug-internal.log
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:init():830] calling init triggers
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'vla': {'type': 'myvla-qwen-224px+mx-mshab', 'vla_id': 'myvla-qwen-224px+mx-mshab', 'base_vlm': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'freeze_vision_backbone': True, 'freeze_llm_backbone': False, 'unfreeze_last_llm_layer': False, 'data_mix': 'bridge', 'shuffle_buffer_size': 256000, 'epochs': 10, 'max_steps': None, 'save_every_n_steps': 25000, 'expected_world_size': 8, 'global_batch_size': 512, 'per_device_batch_size': 8, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'max_grad_norm': 1.0, 'lr_scheduler_type': 'constant', 'warmup_ratio': 0.0, 'train_strategy': 'fsdp-full-shard', 'action_tokenizer': 'extra_action_tokenizer', 'image_sequence_len': 4, 'use_wrist_image': True, 'compress_history': False, 'use_flow_matching': False, 'action_chunk_size': 8, 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'image_window_size': 1}, 'model_type': 'my_vla_qwen', 'data_root_dir': '/home/ubuntu/jianwen-us-midwest-1/tulab/ruisen/.new_maniskill_data', 'run_root_dir': 'myvla_exp', 'pretrained_checkpoint': 'Stanford-ILIAD/prism-qwen25-extra-dinosiglip-224px-0_5b', 'is_resume': False, 'resume_step': None, 'resume_epoch': None, 'run_id': 'bl_multiview_depth_set_table', 'run_id_note': None, 'save_interval': 1000, 'image_aug': False, 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'mshab_vla', 'wandb_entity': 'traysen879-uc-san-diego', 'global_pose': False, 'is_grasped': False, 'qpos': False, 'depth': False, 'segmentation': False, '_wandb': {}}
+2025-08-22 17:55:44,852 INFO    MainThread:4004464 [wandb_init.py:init():871] starting backend
+2025-08-22 17:55:45,099 INFO    MainThread:4004464 [wandb_init.py:init():874] sending inform_init request
+2025-08-22 17:55:45,108 INFO    MainThread:4004464 [wandb_init.py:init():882] backend started and connected
+2025-08-22 17:55:45,114 INFO    MainThread:4004464 [wandb_init.py:init():953] updated telemetry
+2025-08-22 17:55:45,150 INFO    MainThread:4004464 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-22 17:55:45,586 INFO    MainThread:4004464 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-22 17:55:46,161 INFO    MainThread:4004464 [wandb_run.py:_console_start():2458] atexit reg
+2025-08-22 17:55:46,161 INFO    MainThread:4004464 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-08-22 17:55:46,163 INFO    MainThread:4004464 [wandb_run.py:_redirect():2375] Wrapping output streams.
+2025-08-22 17:55:46,163 INFO    MainThread:4004464 [wandb_run.py:_redirect():2398] Redirects installed.
+2025-08-22 17:55:46,173 INFO    MainThread:4004464 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-23 06:27:09,979 INFO    MainThread:4004464 [wandb_run.py:_finish():2224] finishing run traysen879-uc-san-diego/mshab_vla/mg58khw0
+2025-08-23 06:27:09,980 INFO    MainThread:4004464 [wandb_run.py:_atexit_cleanup():2423] got exitcode: 0
+2025-08-23 06:27:09,980 INFO    MainThread:4004464 [wandb_run.py:_restore():2405] restore
+2025-08-23 06:27:09,981 INFO    MainThread:4004464 [wandb_run.py:_restore():2411] restore done
+2025-08-23 06:27:10,444 INFO    MainThread:4004464 [wandb_run.py:_footer_history_summary_info():3903] rendering history
+2025-08-23 06:27:10,446 INFO    MainThread:4004464 [wandb_run.py:_footer_history_summary_info():3935] rendering summary
+2025-08-23 06:27:10,446 INFO    MainThread:4004464 [wandb_run.py:_footer_sync_info():3864] logging synced files

wandb/run-20250822_175544-mg58khw0/run-mg58khw0.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f171964c393d3a2a2a01822280a9c562b11ad093c2cefbe365d066562591d66
+size 21823104