Add files using upload-large-folder tool

config.yaml

@@ -0,0 +1,56 @@
+datasets:
+  vla_data:
+    CoT_prompt: Your task is {instruction}. To identify the key objects for your task.
+      Locate their bounding boxes in [x1,y1,x2,y2] format.
+    action_mode: abs
+    balance_dataset_weights: false
+    balance_trajectory_weights: false
+    data_mix: realv3_droidv2_franka_mix
+    data_root_dir: /project/vonneumann1
+    dataset_py: lerobot_datasets
+    family_specific_action_normalization: true
+    family_specific_metadata: true
+    mix_family_weights: real_v3=1,droid_v2=1
+    per_device_batch_size: 24
+    sequential_step_sampling: false
+    video_backend: torchvision_av
+framework:
+  action_model:
+    action_dim: 7
+    action_hidden_dim: 2560
+    action_model_type: DiT-B
+    future_action_window_size: 15
+    past_action_window_size: 0
+  name: QwenOFT
+  qwenvl:
+    base_vlm: /project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
+output_dir: /project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/results/Checkpoints/0425_realv3_droidv2_subtask_qwen3oft
+run_id: 0425_realv3_droidv2_subtask_qwen3oft
+run_root_dir: /project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/results/Checkpoints
+seed: 42
+trainer:
+  enable_gradient_checkpointing: true
+  eval_interval: 1000
+  freeze_modules: null
+  gradient_accumulation_steps: 1
+  gradient_clipping: 1.0
+  is_resume: false
+  learning_rate:
+    action_model: 0.0001
+    base: 1.0e-05
+    qwen_vl_interface: 1.0e-05
+  logging_frequency: 100
+  lr_scheduler_type: cosine_with_min_lr
+  max_train_steps: 50000
+  num_warmup_steps: 10000
+  optimizer:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  save_interval: 10000
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+wandb_entity: yangsenqiao
+wandb_project: EM-LLaVA

dataset_statistics.json

@@ -0,0 +1,378 @@
+{
+  "family__real_v3": {
+    "action": {
+      "mean": [
+        0.0001020143616187852,
+        -2.8438336812541796e-06,
+        -4.562695648928639e-05,
+        -5.636110745399492e-05,
+        -9.39904301048955e-05,
+        2.4035021220925044e-05,
+        0.5535838961601258
+      ],
+      "std": [
+        0.0044340605017184655,
+        0.004923297240468056,
+        0.007911445123325377,
+        0.008293531573297807,
+        0.01060671336189436,
+        0.011692615051510756,
+        0.44465486464807674
+      ],
+      "max": [
+        0.03620489314198494,
+        0.04532773792743683,
+        0.04355017468333244,
+        0.15797846019268036,
+        0.09877979755401611,
+        0.16858762502670288,
+        1.0
+      ],
+      "min": [
+        -0.03744926676154137,
+        -0.06703328341245651,
+        -0.07337051630020142,
+        -0.11596223711967468,
+        -0.12381929159164429,
+        -0.16848918795585632,
+        0.0
+      ],
+      "q01": [
+        -0.02198706567287445,
+        -0.018244676291942596,
+        -0.043931398540735245,
+        -0.03563353791832924,
+        -0.04498979449272156,
+        -0.04199449345469475,
+        0.0
+      ],
+      "q99": [
+        0.0181239303201437,
+        0.021983295679092407,
+        0.02685590647161007,
+        0.06213546171784401,
+        0.037589989602565765,
+        0.05054198205471039,
+        0.9585910439491272
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        -0.010034098103642464,
+        0.14097123295068742,
+        -0.09451875053346158,
+        -2.182400941848755,
+        -0.026523698493838312,
+        2.257556247711182,
+        0.04985094629228115,
+        -6.554999854415656e-05,
+        0.00019144653633702547,
+        0.00039912204229040075,
+        0.00010078563063871114,
+        -0.00022263232822297143,
+        -0.00027013452199753374,
+        0.0004618760613084306,
+        0.5194364428520203,
+        -0.053295174986124044,
+        0.32030444741249087,
+        0.9750447988510131,
+        -0.05884720273315906,
+        -0.025597028876654805,
+        0.013311849808087575,
+        -0.00014772307469002047,
+        0.00017124121659435333,
+        -9.467979107284917e-05,
+        0.0005717451567761599,
+        -0.0003056184399611084,
+        0.0004661977494833991,
+        0.05535898804664612
+      ],
+      "std": [
+        0.13941935174911055,
+        0.235814846523369,
+        0.20177433104748257,
+        0.31738078653451646,
+        0.1907652055517865,
+        0.28316418242072144,
+        0.32730431727018655,
+        0.04502095577777268,
+        0.22103854943862805,
+        0.1277183320685496,
+        0.26735176611738093,
+        0.11402699967529559,
+        0.24877026358511437,
+        0.14352576798609462,
+        0.08617109311511709,
+        0.08452529534008903,
+        0.09431220241046796,
+        0.08607295676504408,
+        0.16164004391855352,
+        0.06940346688256428,
+        0.08148306542135049,
+        0.06534789269151228,
+        0.07223249480799422,
+        0.11418239190298414,
+        0.11447460721841322,
+        0.16107240169019812,
+        0.1725334350554518,
+        0.04446671179130432
+      ],
+      "max": [
+        0.4881610870361328,
+        0.8126450181007385,
+        0.4563447833061218,
+        -1.0394330024719238,
+        0.8490543961524963,
+        3.083712100982666,
+        1.4997336864471436,
+        0.7826632857322693,
+        1.6545618772506714,
+        1.4005711078643799,
+        1.6076738834381104,
+        1.7229373455047607,
+        1.6480886936187744,
+        1.6966770887374878,
+        0.7470663189888,
+        0.23474355041980743,
+        0.6643985509872437,
+        0.9999979734420776,
+        0.9079863429069519,
+        0.24088943004608154,
+        0.745482861995697,
+        0.485066294670105,
+        0.8514209985733032,
+        0.9038169980049133,
+        1.8596763610839844,
+        1.771505355834961,
+        2.2467849254608154,
+        0.10000000149011612
+      ],
+      "min": [
+        -0.39643630385398865,
+        -0.7641158699989319,
+        -0.8546575307846069,
+        -2.913532018661499,
+        -1.3668692111968994,
+        1.1820083856582642,
+        -1.8731739521026611,
+        -0.7894002199172974,
+        -1.4145859479904175,
+        -1.4128144979476929,
+        -1.5183827877044678,
+        -1.4721919298171997,
+        -1.7565959692001343,
+        -1.6417104005813599,
+        0.2901875674724579,
+        -0.3145847022533417,
+        0.19108672440052032,
+        -0.7037295699119568,
+        -0.7020035982131958,
+        -0.3718150854110718,
+        -0.48518362641334534,
+        -0.4672047197818756,
+        -0.6322252154350281,
+        -0.6577076315879822,
+        -1.661864161491394,
+        -1.378888726234436,
+        -2.1491503715515137,
+        0.0
+      ],
+      "q01": [
+        -0.3224031925201416,
+        -0.43004000186920166,
+        -0.7309625148773193,
+        -2.8752927780151367,
+        -0.9317355155944824,
+        1.4511760473251343,
+        -1.166956901550293,
+        -0.22034505009651184,
+        -0.9354689121246338,
+        -0.5218945145606995,
+        -1.0582122802734375,
+        -0.5365700721740723,
+        -1.2749916315078735,
+        -0.5920586585998535,
+        0.33311885595321655,
+        -0.2575554847717285,
+        0.19956959784030914,
+        0.7451193928718567,
+        -0.5359444618225098,
+        -0.3100806176662445,
+        -0.34559693932533264,
+        -0.2991495430469513,
+        -0.32358548045158386,
+        -0.3659352958202362,
+        -0.5153824687004089,
+        -0.5692487955093384,
+        -0.7244541645050049,
+        0.0
+      ],
+      "q99": [
+        0.3976858854293823,
+        0.7480529546737671,
+        0.3511078953742981,
+        -1.4764916896820068,
+        0.6248592734336853,
+        2.9200379848480225,
+        1.0268296003341675,
+        0.2207048386335373,
+        0.7570937871932983,
+        0.4898764193058014,
+        1.4297515153884888,
+        0.8266521096229553,
+        0.8434298038482666,
+        0.5895139575004578,
+        0.716124951839447,
+        0.1764630824327469,
+        0.5704453587532043,
+        0.9997912049293518,
+        0.599913477897644,
+        0.1716972142457962,
+        0.4558342695236206,
+        0.2602992653846741,
+        0.29712045192718506,
+        0.5703409314155579,
+        0.891438901424408,
+        0.654464602470398,
+        0.6311540603637695,
+        0.0958591029047966
+      ]
+    },
+    "num_transitions": 116479,
+    "num_trajectories": 340
+  },
+  "family__droid_v2": {
+    "action": {
+      "mean": [
+        0.0005689284880645573,
+        -2.3364036678685807e-06,
+        0.0004740336153190583,
+        -0.00011216480197617784,
+        0.0004274287784937769,
+        0.0002528380136936903,
+        0.3626807928085327
+      ],
+      "std": [
+        0.004631359595805407,
+        0.0046998281031847,
+        0.005355845205485821,
+        0.012149380519986153,
+        0.013062640093266964,
+        0.01614941470324993,
+        0.425840824842453
+      ],
+      "max": [
+        0.15221983194351196,
+        0.16005533933639526,
+        0.1329832226037979,
+        0.2810741364955902,
+        0.2754988968372345,
+        0.30190709233283997,
+        1.0
+      ],
+      "min": [
+        -0.13691653311252594,
+        -0.12317865341901779,
+        -0.13230597972869873,
+        -0.2870335578918457,
+        -0.26662254333496094,
+        -0.26537832617759705,
+        0.0
+      ],
+      "q01": [
+        -0.012801917269825935,
+        -0.014855321496725082,
+        -0.01611165702342987,
+        -0.03964243084192276,
+        -0.0404227152466774,
+        -0.05156254395842552,
+        0.0
+      ],
+      "q99": [
+        0.015180022455751896,
+        0.01495266705751419,
+        0.015406008809804916,
+        0.03957250341773033,
+        0.04320498928427696,
+        0.051785826683044434,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        0.3626807928085327,
+        -0.003969562239944935,
+        0.17508485913276672,
+        0.31842565536499023,
+        -0.09132317453622818,
+        -0.05298471450805664,
+        0.3626807928085327
+      ],
+      "std": [
+        0.20250748097896576,
+        0.17296893894672394,
+        0.16208088397979736,
+        2.408919334411621,
+        0.333966463804245,
+        0.7269497513771057,
+        0.425840824842453
+      ],
+      "max": [
+        0.9941494464874268,
+        0.8930133581161499,
+        1.0969828367233276,
+        3.1415927410125732,
+        1.5705928802490234,
+        3.141592502593994,
+        1.0
+      ],
+      "min": [
+        -0.2344866245985031,
+        -0.9323146939277649,
+        -0.3867732882499695,
+        -3.141592502593994,
+        -1.5703768730163574,
+        -3.141563653945923,
+        0.0
+      ],
+      "q01": [
+        0.266904354095459,
+        -0.45883017778396606,
+        -0.17589999735355377,
+        -3.1375010013580322,
+        -1.2102854251861572,
+        -2.137455463409424,
+        0.0
+      ],
+      "q99": [
+        0.8327664136886597,
+        0.45490509271621704,
+        0.7126270532608032,
+        3.137559413909912,
+        0.8736662864685059,
+        2.017656087875366,
+        1.0
+      ]
+    },
+    "num_transitions": 35692400,
+    "num_trajectories": 152986
+  }
+}

final_model/pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d0fca47eb9463c4c8cb8791c09e92ede57913cd9bf2af4b4c73b5669793215d
+size 9785052178

run_franka_trainV3Data_DroidV2.sh

@@ -0,0 +1,184 @@
+#!/bin/bash
+#SBATCH --job-name=franka_v3_droidv2_mix
+#SBATCH --partition=vonneumann
+#SBATCH --account=vonneumann1
+#SBATCH --nodes=1
+#SBATCH --gpus-per-node=8
+#SBATCH --cpus-per-gpu=24
+#SBATCH --ntasks-per-node=1
+#SBATCH --output=logs/franka_v3_droidv2_mix_%j.out
+#SBATCH --error=logs/franka_v3_droidv2_mix_%j.err
+
+set -eo pipefail
+
+source /project/vonneumann1/sqyang/.bashrc
+conda activate starVLA
+
+export NCCL_SOCKET_IFNAME=bond0
+export NCCL_IB_HCA=mlx5_2,mlx5_3
+export NCCL_BLOCKING_WAIT=1
+export NCCL_ASYNC_ERROR_HANDLING=1
+export NCCL_TIMEOUT=10000
+
+SCRIPT_REL_PATH=examples/RealRobot/run_franka_trainV3Data_DroidV2.sh
+if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/starVLA" ]]; then
+  REPO_ROOT="${SLURM_SUBMIT_DIR}"
+else
+  SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+  REPO_ROOT=$(cd -- "${SCRIPT_DIR}/../.." && pwd)
+fi
+
+if [[ ! -d "${REPO_ROOT}/starVLA" ]]; then
+  echo "Resolved REPO_ROOT is invalid: ${REPO_ROOT}" >&2
+  exit 1
+fi
+
+SCRIPT_SOURCE="${REPO_ROOT}/${SCRIPT_REL_PATH}"
+cd "${REPO_ROOT}"
+
+export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}"
+
+mkdir -p logs
+
+MASTER_ADDR=$(scontrol show hostnames "${SLURM_JOB_NODELIST:-$(hostname)}" | head -n 1)
+MASTER_PORT=29510
+export MASTER_ADDR MASTER_PORT
+
+NNODES=${SLURM_NNODES:-1}
+GPUS_PER_NODE=${GPUS_PER_NODE:-8}
+TOTAL_GPUS=$((NNODES * GPUS_PER_NODE))
+
+Framework_name=${FRAMEWORK_NAME:-QwenOFT}
+freeze_module_list=${FREEZE_MODULE_LIST:-}
+base_vlm=${BASE_VLM:-"${REPO_ROOT}/playground/Pretrained_models/Qwen3-VL-4B-Instruct"}
+config_yaml=${CONFIG_YAML:-"${REPO_ROOT}/examples/RealRobot/starvla_cotrain_franka.yaml"}
+data_root_dir=${DATA_ROOT_DIR:-/project/vonneumann1}
+data_mix=${DATA_MIX:-realv3_droidv2_franka_mix}
+run_root_dir=${RUN_ROOT_DIR:-"${REPO_ROOT}/results/Checkpoints"}
+run_id=${RUN_ID:-0425_realv3_droidv2_subtask_qwen3oft}
+warmup_steps=${WARMUP_STEPS:-10000}
+per_device_batch_size=${PER_DEVICE_BATCH_SIZE:-24}
+max_train_steps=${MAX_TRAIN_STEPS:-50000}
+save_interval=${SAVE_INTERVAL:-10000}
+logging_frequency=${LOGGING_FREQUENCY:-100}
+eval_interval=${EVAL_INTERVAL:-1000}
+
+# Family-level sampling ratio. Effective side probability is proportional to these two values.
+# Example: REAL_FAMILY_WEIGHT=1 DROID_FAMILY_WEIGHT=3 means RealV3:DROIDV2 = 1:3.
+REAL_FAMILY_WEIGHT=${REAL_FAMILY_WEIGHT:-1}
+DROID_FAMILY_WEIGHT=${DROID_FAMILY_WEIGHT:-1}
+MIX_FAMILY_WEIGHTS=${MIX_FAMILY_WEIGHTS:-real_v3=${REAL_FAMILY_WEIGHT},droid_v2=${DROID_FAMILY_WEIGHT}}
+
+BALANCE_DATASET_WEIGHTS=${BALANCE_DATASET_WEIGHTS:-false}
+BALANCE_TRAJECTORY_WEIGHTS=${BALANCE_TRAJECTORY_WEIGHTS:-false}
+FAMILY_SPECIFIC_ACTION_NORMALIZATION=${FAMILY_SPECIFIC_ACTION_NORMALIZATION:-true}
+FAMILY_SPECIFIC_METADATA=${FAMILY_SPECIFIC_METADATA:-true}
+ACTION_MODE=${ACTION_MODE:-abs}
+
+PRECOMPUTE_STATS=${PRECOMPUTE_STATS:-1}
+STATS_WORKERS=${STATS_WORKERS:-16}
+STATS_TMP_DIR=${STATS_TMP_DIR:-}
+FORCE_REBUILD_STATS=${FORCE_REBUILD_STATS:-0}
+
+stats_datasets=(
+  "/project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/playground/Datasets/RealworldDataset-V3/lerobot_output/clean_table_lerobot"
+  "/project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/playground/Datasets/RealworldDataset-V3/lerobot_output/cube_up_lerobot"
+  "/project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/playground/Datasets/RealworldDataset-V3/lerobot_output/pick_carrots_out_lerobot"
+  "/project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/playground/Datasets/RealworldDataset-V3/lerobot_output/use_pot_lerobot"
+  "/project/vonneumann1/sqyang/project/EM-LLaVA/starVLA/playground/Datasets/RealworldDataset-V3/lerobot_output/use_spoon_lerobot"
+  "/project/vonneumann1/datasets/RoboInter-Data/Annotation_with_action_lerobotv21/lerobot_droid_anno"
+)
+
+output_dir=${run_root_dir}/${run_id}
+mkdir -p "${output_dir}"
+if [[ -f "${SCRIPT_SOURCE}" ]]; then
+  cp "${SCRIPT_SOURCE}" "${output_dir}/"
+else
+  cp "$0" "${output_dir}/"
+fi
+
+echo "============================================"
+echo "Job ID                : ${SLURM_JOB_ID:-manual}"
+echo "Nodes                 : ${SLURM_JOB_NODELIST:-$(hostname)}"
+echo "NNODES                : ${NNODES}"
+echo "GPUS/NODE             : ${GPUS_PER_NODE}"
+echo "TOTAL GPUS            : ${TOTAL_GPUS}"
+echo "MASTER_ADDR           : ${MASTER_ADDR}"
+echo "MASTER_PORT           : ${MASTER_PORT}"
+echo "Framework             : ${Framework_name}"
+echo "Data root             : ${data_root_dir}"
+echo "Data mix              : ${data_mix}"
+echo "Mix family weights    : ${MIX_FAMILY_WEIGHTS}"
+echo "Balance dataset       : ${BALANCE_DATASET_WEIGHTS}"
+echo "Balance trajectory    : ${BALANCE_TRAJECTORY_WEIGHTS}"
+echo "Family-specific stats : ${FAMILY_SPECIFIC_ACTION_NORMALIZATION}"
+echo "Family-specific meta  : ${FAMILY_SPECIFIC_METADATA}"
+echo "Action mode           : ${ACTION_MODE}"
+echo "============================================"
+
+if [[ "${PRECOMPUTE_STATS}" == "1" ]]; then
+  for dataset_root in "${stats_datasets[@]}"; do
+    if [[ ! -d "${dataset_root}" ]]; then
+      echo "Stats dataset root not found: ${dataset_root}" >&2
+      exit 1
+    fi
+
+    stats_output_path="${dataset_root}/meta/stats_gr00t.json"
+    if [[ ! -f "${stats_output_path}" || "${FORCE_REBUILD_STATS}" == "1" ]]; then
+      echo "Precomputing dataset statistics offline for ${dataset_root}"
+      stats_cmd=(
+        python "${REPO_ROOT}/scripts/build_lerobot_stats_parallel.py"
+        "${dataset_root}"
+        --workers "${STATS_WORKERS}"
+      )
+      if [[ -n "${STATS_TMP_DIR}" ]]; then
+        stats_cmd+=(--tmp-dir "${STATS_TMP_DIR}")
+      fi
+      if [[ "${FORCE_REBUILD_STATS}" == "1" ]]; then
+        stats_cmd+=(--overwrite)
+      fi
+      "${stats_cmd[@]}"
+    else
+      echo "Reusing existing dataset statistics: ${stats_output_path}"
+    fi
+  done
+fi
+
+launch_cmd=(
+  accelerate launch
+  --config_file "${REPO_ROOT}/starVLA/config/deepseeds/deepspeed_zero2.yaml"
+  --main_process_ip "${MASTER_ADDR}"
+  --main_process_port "${MASTER_PORT}"
+  --machine_rank 0
+  --num_machines "${NNODES}"
+  --num_processes "${TOTAL_GPUS}"
+  "${REPO_ROOT}/starVLA/training/train_starvla.py"
+  --config_yaml "${config_yaml}"
+  --framework.name "${Framework_name}"
+  --framework.qwenvl.base_vlm "${base_vlm}"
+  --datasets.vla_data.data_root_dir "${data_root_dir}"
+  --datasets.vla_data.data_mix "${data_mix}"
+  --datasets.vla_data.mix_family_weights "${MIX_FAMILY_WEIGHTS}"
+  --datasets.vla_data.balance_dataset_weights "${BALANCE_DATASET_WEIGHTS}"
+  --datasets.vla_data.balance_trajectory_weights "${BALANCE_TRAJECTORY_WEIGHTS}"
+  --datasets.vla_data.family_specific_action_normalization "${FAMILY_SPECIFIC_ACTION_NORMALIZATION}"
+  --datasets.vla_data.family_specific_metadata "${FAMILY_SPECIFIC_METADATA}"
+  --datasets.vla_data.action_mode "${ACTION_MODE}"
+  --datasets.vla_data.per_device_batch_size "${per_device_batch_size}"
+  --trainer.freeze_modules "${freeze_module_list}"
+  --trainer.max_train_steps "${max_train_steps}"
+  --trainer.num_warmup_steps "${warmup_steps}"
+  --trainer.save_interval "${save_interval}"
+  --trainer.logging_frequency "${logging_frequency}"
+  --trainer.eval_interval "${eval_interval}"
+  --run_root_dir "${run_root_dir}"
+  --run_id "${run_id}"
+  --wandb_project EM-LLaVA
+  --wandb_entity yangsenqiao
+)
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+  srun --kill-on-bad-exit=1 "${launch_cmd[@]}"
+else
+  "${launch_cmd[@]}"
+fi

summary.jsonl

@@ -0,0 +1,5 @@
+{"steps": 10000}
+{"steps": 20000}
+{"steps": 30000}
+{"steps": 40000}
+{"steps": 50000}