Add files using upload-large-folder tool

config.yaml

@@ -0,0 +1,54 @@
+datasets:
+  vla_data:
+    action_mode: abs
+    data_mix: robotwin_clean_32
+    data_root_dir: /project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/playground/Datasets/RoboTwin-Clean
+    dataset_py: lerobot_datasets
+    image_size:
+    - 224
+    - 224
+    per_device_batch_size: 16
+    sequential_step_sampling: false
+    video_backend: pyav
+framework:
+  action_model:
+    action_dim: 14
+    action_hidden_dim: 2560
+    action_model_type: DiT-B
+    future_action_window_size: 31
+    past_action_window_size: 0
+  name: QwenOFT
+  qwenvl:
+    base_vlm: /project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
+output_dir: /project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/results/Checkpoints/0328robotwin_from_robocoin_1x100k_qwenonly_randomaction_horizon32_frombalance_WM2
+run_id: 0328robotwin_from_robocoin_1x100k_qwenonly_randomaction_horizon32_frombalance_WM2
+run_root_dir: /project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/results/Checkpoints
+seed: 42
+trainer:
+  enable_gradient_checkpointing: true
+  eval_interval: 1000
+  freeze_modules: true
+  gradient_accumulation_steps: 1
+  gradient_clipping: 1.0
+  is_resume: false
+  learning_rate:
+    action_model: 0.0001
+    base: 1.0e-05
+    qwen_vl_interface: 1.0e-05
+  logging_frequency: 100
+  lr_scheduler_type: cosine_with_min_lr
+  max_train_steps: 50000
+  num_warmup_steps: 5000
+  optimizer:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  pretrained_checkpoint: /project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/results/Checkpoints/0330_robocoin_sq_all_qwen3OFT_WorldModelV2_freezeVisionLLMHalf_balanceDataset/checkpoints/steps_90000_pytorch_model.pt
+  reload_modules: qwen_vl_interface
+  save_interval: 10000
+  scheduler_specific_kwargs:
+    min_lr: 5.0e-07
+wandb_entity: yangsenqiao
+wandb_project: EM-LLaVA

dataset_statistics.json

@@ -0,0 +1,218 @@
+{
+  "new_embodiment": {
+    "action": {
+      "mean": [
+        -0.21901101492345337,
+        1.0504990947246553,
+        0.7661802005767822,
+        -0.3331204185634851,
+        0.06365180747252454,
+        -0.06979782831855118,
+        0.23065026089549062,
+        1.1413198339939117,
+        0.8520238375663758,
+        -0.3788263291306793,
+        -0.025514634509745525,
+        0.04028999065980314,
+        0.6806974941492078,
+        0.6566390764713288
+      ],
+      "std": [
+        0.3882080026926371,
+        0.9962431978470905,
+        0.7826509819359658,
+        0.6701073679280808,
+        0.25934216358594864,
+        0.6261345167669774,
+        0.3245623193152152,
+        1.0132390536926115,
+        0.8113444755250797,
+        0.7242207074420685,
+        0.26989906331402447,
+        0.6765045276556505,
+        0.4472407649627974,
+        0.45477049309587164
+      ],
+      "max": [
+        5.681515216827393,
+        3.8880207538604736,
+        4.500889301300049,
+        1.789766788482666,
+        1.5632697343826294,
+        4.389739513397217,
+        1.3892723321914673,
+        3.242604970932007,
+        3.6588551998138428,
+        1.9530924558639526,
+        1.397123098373413,
+        3.503765106201172,
+        1.0,
+        1.0
+      ],
+      "min": [
+        -7.340834140777588,
+        -0.0003164021181873977,
+        -0.1190902441740036,
+        -1.9268131256103516,
+        -1.4216828346252441,
+        -6.232340335845947,
+        -6.316576957702637,
+        -0.5742834806442261,
+        -0.005311839282512665,
+        -1.9893786907196045,
+        -2.1285502910614014,
+        -6.269529342651367,
+        0.0,
+        0.0
+      ],
+      "q01": [
+        -7.334665780067444,
+        -5.257390398583084e-07,
+        -2.2967718905420043e-05,
+        -1.841589093208313,
+        -1.1798312604427337,
+        -5.643983192443847,
+        -5.995517673492431,
+        -0.36572347044944764,
+        -2.81171942333458e-05,
+        -1.8467556238174438,
+        -1.4545854091644288,
+        -5.945669202804565,
+        0.0,
+        0.0
+      ],
+      "q99": [
+        5.016749763488781,
+        3.4728481292724593,
+        4.225283479690551,
+        1.7033938264846804,
+        1.4612830877304077,
+        2.8437052965164185,
+        1.258314609527588,
+        2.898959903717041,
+        3.12267804145813,
+        1.750970482826233,
+        1.280245304107666,
+        3.4143478631973267,
+        1.0,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        -0.21762139897793528,
+        1.0445352470874787,
+        0.7618057584762571,
+        -0.33136719916015855,
+        0.06322407866467984,
+        -0.06871131842024625,
+        0.22887468233704572,
+        1.1339007771015168,
+        0.8465183085203172,
+        -0.37649038800038415,
+        -0.02524788704147795,
+        0.03936799738556147,
+        0.682512201219797,
+        0.6585012382268904
+      ],
+      "std": [
+        0.38712398853080604,
+        0.9965709002640186,
+        0.7823577599579604,
+        0.6681911921924909,
+        0.2584405378948888,
+        0.6233249305188155,
+        0.323771915217167,
+        1.0139910538683092,
+        0.8113518216713491,
+        0.7220370403976613,
+        0.2687367916036712,
+        0.6730492302105592,
+        0.4465031278188237,
+        0.4541251057289932
+      ],
+      "max": [
+        5.681515216827393,
+        3.8880207538604736,
+        4.500889301300049,
+        1.789766788482666,
+        1.5632697343826294,
+        4.389739513397217,
+        1.3892723321914673,
+        3.242604970932007,
+        3.6588551998138428,
+        1.9530924558639526,
+        1.397123098373413,
+        3.503765106201172,
+        1.0,
+        1.0
+      ],
+      "min": [
+        -7.340834140777588,
+        -0.0003164021181873977,
+        -0.1190902441740036,
+        -1.9268131256103516,
+        -1.4216828346252441,
+        -6.232340335845947,
+        -6.316576957702637,
+        -0.5742834806442261,
+        -0.005311839282512665,
+        -1.9893786907196045,
+        -2.1285502910614014,
+        -6.269529342651367,
+        0.0,
+        0.0
+      ],
+      "q01": [
+        -7.334665780067444,
+        -5.257390398583084e-07,
+        -2.2967718905420043e-05,
+        -1.841589093208313,
+        -1.1798312604427337,
+        -5.543431510925292,
+        -5.912737417221069,
+        -0.36572347044944764,
+        -2.81171942333458e-05,
+        -1.8467556238174438,
+        -1.440324923992157,
+        -5.8623305034637445,
+        0.0,
+        0.0
+      ],
+      "q99": [
+        4.902149534225477,
+        3.4690011501312243,
+        4.224453401565551,
+        1.7033938264846804,
+        1.4611416244506834,
+        2.8405991554260255,
+        1.258314609527588,
+        2.898959903717041,
+        3.12267804145813,
+        1.7416918659210197,
+        1.280245304107666,
+        3.4138864517211913,
+        1.0,
+        1.0
+      ]
+    },
+    "num_transitions": 549787,
+    "num_trajectories": 2500
+  }
+}

final_model/pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80b11c6ab0b47f5a6cc417985b884f691711e7ee3a323bfb7925bd13ae1f5c78
+size 9785123858

run_robocoin_finetune_robotwin_abs_random_action_frombalance.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+#SBATCH --job-name=robotwin_qwra
+#SBATCH --partition=vonneumann
+#SBATCH --account=vonneumann1
+#SBATCH --nodes=1
+#SBATCH --gpus-per-node=8
+#SBATCH --cpus-per-gpu=24
+#SBATCH --ntasks-per-node=1
+#SBATCH --output=logs/robotwin_qwra_%j.out
+#SBATCH --error=logs/robotwin_qwra_%j.err
+
+# ============================================================
+#  Robotwin abs finetune from RoboCOIN checkpoint
+#  Load only Qwen3-VL weights, keep action head randomly initialized.
+#  Submit: sbatch examples/Robocoin/train_files/run_robocoin_finetune_robotwin_abs_random_action_frombalance.sh
+# ============================================================
+
+set -eo pipefail
+
+source /project/vonneumann1/sqyang/.bashrc
+conda activate starVLA
+
+export NCCL_SOCKET_IFNAME=bond0
+export NCCL_IB_HCA=mlx5_2,mlx5_3
+export NCCL_BLOCKING_WAIT=1
+export NCCL_ASYNC_ERROR_HANDLING=1
+export NCCL_TIMEOUT=3600
+
+SCRIPT_REL_PATH=examples/Robocoin/train_files/run_robocoin_finetune_robotwin_abs_random_action_frombalance.sh
+if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/starVLA" ]]; then
+  REPO_ROOT="${SLURM_SUBMIT_DIR}"
+else
+  SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+  REPO_ROOT=$(cd -- "${SCRIPT_DIR}/../../.." && pwd)
+fi
+
+if [[ ! -d "${REPO_ROOT}/starVLA" ]]; then
+  echo "Resolved REPO_ROOT is invalid: ${REPO_ROOT}" >&2
+  exit 1
+fi
+
+SCRIPT_SOURCE="${REPO_ROOT}/${SCRIPT_REL_PATH}"
+cd "${REPO_ROOT}"
+
+export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}"
+
+mkdir -p logs
+
+MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
+MASTER_PORT=29500
+export MASTER_ADDR MASTER_PORT
+
+NNODES=${SLURM_NNODES:-1}
+GPUS_PER_NODE=8
+TOTAL_GPUS=$((NNODES * GPUS_PER_NODE))
+
+echo "============================================"
+echo "Job ID                : $SLURM_JOB_ID"
+echo "Nodes                 : $SLURM_NODELIST"
+echo "NNODES                : $NNODES"
+echo "GPUS/NODE             : $GPUS_PER_NODE"
+echo "TOTAL GPUS            : $TOTAL_GPUS"
+echo "MASTER_ADDR           : $MASTER_ADDR"
+echo "MASTER_PORT           : $MASTER_PORT"
+echo "============================================"
+
+Framework_name=QwenOFT
+freeze_module_list=''
+reload_modules='qwen_vl_interface'
+base_vlm="${REPO_ROOT}/playground/Pretrained_models/Qwen3-VL-4B-Instruct"
+config_yaml="${REPO_ROOT}/examples/Robotwin/train_files/starvla_cotrain_robotwin_abs_horizon32.yaml"
+run_root_dir="${REPO_ROOT}/results/Checkpoints"
+data_mix=robotwin_clean_32
+data_root_dir="${REPO_ROOT}/playground/Datasets/RoboTwin-Clean"
+lr_base=1e-05
+lr_qwen=1e-05
+lr_action=1e-04
+pretrained_checkpoint="${REPO_ROOT}/results/Checkpoints/0328_robocoin_sq_all_qwen3OFT_freezeVisionLLMHalf_balanceDataset/checkpoints/steps_100000_pytorch_model.pt"
+run_id=0328robotwin_clean_32_qwen3OFT_ft_abs_from_robocoin_1x100k_qwenonly_randomaction_horizon32_frombalance
+
+output_dir=${run_root_dir}/${run_id}
+mkdir -p "${output_dir}"
+if [[ -f "${SCRIPT_SOURCE}" ]]; then
+  cp "${SCRIPT_SOURCE}" "${output_dir}/"
+else
+  cp "$0" "${output_dir}/"
+fi
+
+
+echo "Pretrained checkpoint : ${pretrained_checkpoint}"
+echo "Reload modules        : ${reload_modules}"
+echo "Freeze modules        : ${freeze_module_list:-<none>}"
+echo "Learning rates        : base=${lr_base}, qwen=${lr_qwen}, action=${lr_action}"
+
+srun --kill-on-bad-exit=1 bash -c '
+  accelerate launch \
+    --config_file '"${REPO_ROOT}"'/starVLA/config/deepseeds/deepspeed_zero2.yaml \
+    --main_process_ip '"$MASTER_ADDR"' \
+    --main_process_port '"$MASTER_PORT"' \
+    --machine_rank $SLURM_PROCID \
+    --num_machines '"$NNODES"' \
+    --num_processes '"$TOTAL_GPUS"' \
+    '"${REPO_ROOT}"'/starVLA/training/train_starvla.py \
+    --config_yaml '"$config_yaml"' \
+    --framework.name '"$Framework_name"' \
+    --framework.qwenvl.base_vlm '"$base_vlm"' \
+    --datasets.vla_data.per_device_batch_size 16 \
+    --datasets.vla_data.data_mix '"$data_mix"' \
+    --datasets.vla_data.data_root_dir '"$data_root_dir"' \
+    --trainer.freeze_modules '"$freeze_module_list"' \
+    --trainer.pretrained_checkpoint '"$pretrained_checkpoint"' \
+    --trainer.reload_modules '"$reload_modules"' \
+    --trainer.learning_rate.base '"$lr_base"' \
+    --trainer.learning_rate.qwen_vl_interface '"$lr_qwen"' \
+    --trainer.learning_rate.action_model '"$lr_action"' \
+    --trainer.max_train_steps 50000 \
+    --trainer.save_interval 10000 \
+    --trainer.logging_frequency 100 \
+    --trainer.eval_interval 1000 \
+    --run_root_dir '"$run_root_dir"' \
+    --run_id '"$run_id"' \
+    --wandb_project EM-LLaVA \
+    --wandb_entity yangsenqiao
+'

summary.jsonl

@@ -0,0 +1,5 @@
+{"steps": 10000}
+{"steps": 20000}
+{"steps": 30000}
+{"steps": 40000}
+{"steps": 50000}