Upload folder using huggingface_hub
Browse files- flappy_fix_latency_3_200ep_7k2steps_single_baseline/.uploaded +0 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.full.yaml +2 -3
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.yaml +1 -31
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/hydra/.hydra/config.yaml +2 -3
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/hydra/.hydra/hydra.yaml +3 -4
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/hydra/.hydra/overrides.yaml +2 -3
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/hydra/train_starvla_hydra.log +312 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/debug-internal.log +0 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/debug.log +23 -24
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/config.yaml +327 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/output.log +35 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/requirements.txt +237 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/wandb-metadata.json +305 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/wandb-summary.json +1 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/logs/debug-core.log +14 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/logs/debug-internal.log +16 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/logs/debug.log +21 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/run-ki7n3drp.wandb +0 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/config.yaml +327 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/output.log +53 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/requirements.txt +237 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/wandb-metadata.json +305 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/wandb-summary.json +1 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/logs/debug-core.log +14 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/logs/debug-internal.log +16 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/logs/debug.log +21 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/run-p3bswjax.wandb +0 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/config.yaml +329 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/output.log +36 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/requirements.txt +237 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/wandb-metadata.json +305 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/wandb-summary.json +1 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug-core.log +19 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug-internal.log +25 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug.log +24 -0
- flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/run-m3s24v9k.wandb +0 -0
flappy_fix_latency_3_200ep_7k2steps_single_baseline/.uploaded
ADDED
|
File without changes
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.full.yaml
CHANGED
|
@@ -3,7 +3,6 @@ framework:
|
|
| 3 |
qwenvl:
|
| 4 |
base_vlm: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 5 |
attn_implementation: flash_attention_2
|
| 6 |
-
flex_backend: triton
|
| 7 |
enable_gradient_checkpointing: true
|
| 8 |
action_model:
|
| 9 |
action_model_type: MLP
|
|
@@ -132,8 +131,8 @@ trainer:
|
|
| 132 |
gradient_accumulation_steps: 16
|
| 133 |
distributed_backend: deepspeed
|
| 134 |
is_resume: true
|
| 135 |
-
pretrained_checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/
|
| 136 |
-
resume_step:
|
| 137 |
reload_modules: null
|
| 138 |
optimizer:
|
| 139 |
name: AdamW
|
|
|
|
| 3 |
qwenvl:
|
| 4 |
base_vlm: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 5 |
attn_implementation: flash_attention_2
|
|
|
|
| 6 |
enable_gradient_checkpointing: true
|
| 7 |
action_model:
|
| 8 |
action_model_type: MLP
|
|
|
|
| 131 |
gradient_accumulation_steps: 16
|
| 132 |
distributed_backend: deepspeed
|
| 133 |
is_resume: true
|
| 134 |
+
pretrained_checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 135 |
+
resume_step: 4000
|
| 136 |
reload_modules: null
|
| 137 |
optimizer:
|
| 138 |
name: AdamW
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.yaml
CHANGED
|
@@ -17,7 +17,6 @@ datasets:
|
|
| 17 |
eval_data_mix: flappy_train__bridge__val
|
| 18 |
latency_curriculum:
|
| 19 |
enabled: false
|
| 20 |
-
obs_image_size: null
|
| 21 |
per_device_batch_size: 8
|
| 22 |
framework:
|
| 23 |
action_model:
|
|
@@ -41,35 +40,14 @@ framework:
|
|
| 41 |
output_dir: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline
|
| 42 |
rl_games:
|
| 43 |
env_eval:
|
| 44 |
-
distributed_mode: rank_sharded
|
| 45 |
enabled: true
|
| 46 |
eval_backend: latency_bench
|
| 47 |
-
eval_parallel_envs: 5
|
| 48 |
-
fixed_episode_seeds: true
|
| 49 |
-
image_size: 224
|
| 50 |
-
image_transform: raw_rgb
|
| 51 |
-
latency:
|
| 52 |
-
prompt_map_path: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps/flappy_train__bridge/latency_prompt_map.json
|
| 53 |
-
mid_train:
|
| 54 |
-
enabled: true
|
| 55 |
-
interval_steps: 250
|
| 56 |
-
latencies:
|
| 57 |
-
- 3
|
| 58 |
-
max_steps_per_episode: 3600
|
| 59 |
-
num_episodes: 20
|
| 60 |
prompt_mode: raw
|
| 61 |
-
seed: 42
|
| 62 |
-
model_alias: openvla
|
| 63 |
-
task: flappy
|
| 64 |
run_id: flappy_fix_latency_3_200ep_7k2steps_single_baseline
|
| 65 |
run_root_dir: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 66 |
seed: 42
|
| 67 |
trainer:
|
| 68 |
distributed_backend: deepspeed
|
| 69 |
-
eval_action_classification: false
|
| 70 |
-
eval_action_classification_interval: null
|
| 71 |
-
eval_interval: 250
|
| 72 |
-
eval_num_batches: 50
|
| 73 |
freeze_llm_layers: []
|
| 74 |
freeze_modules: ''
|
| 75 |
freeze_tied_embedding: false
|
|
@@ -80,7 +58,6 @@ trainer:
|
|
| 80 |
action_model: 0.0001
|
| 81 |
base: 2.0e-05
|
| 82 |
qwen_vl_interface: 1.0e-05
|
| 83 |
-
logging_frequency: 1
|
| 84 |
lr_scheduler_type: cosine_with_min_lr
|
| 85 |
max_train_steps: 4000
|
| 86 |
num_warmup_steps: 100
|
|
@@ -91,13 +68,6 @@ trainer:
|
|
| 91 |
eps: 1.0e-08
|
| 92 |
fused: true
|
| 93 |
weight_decay: 1.0e-08
|
| 94 |
-
|
| 95 |
-
pretrained_checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_2000_state
|
| 96 |
-
profile_timing:
|
| 97 |
-
enabled: false
|
| 98 |
-
resume_step: 2000
|
| 99 |
-
save_interval: 500
|
| 100 |
scheduler_specific_kwargs:
|
| 101 |
min_lr: 1.0e-06
|
| 102 |
-
wandb_entity: zihanwang-ai-northwestern-university
|
| 103 |
-
wandb_project: starVLA_rl_games
|
|
|
|
| 17 |
eval_data_mix: flappy_train__bridge__val
|
| 18 |
latency_curriculum:
|
| 19 |
enabled: false
|
|
|
|
| 20 |
per_device_batch_size: 8
|
| 21 |
framework:
|
| 22 |
action_model:
|
|
|
|
| 40 |
output_dir: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline
|
| 41 |
rl_games:
|
| 42 |
env_eval:
|
|
|
|
| 43 |
enabled: true
|
| 44 |
eval_backend: latency_bench
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
prompt_mode: raw
|
|
|
|
|
|
|
|
|
|
| 46 |
run_id: flappy_fix_latency_3_200ep_7k2steps_single_baseline
|
| 47 |
run_root_dir: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 48 |
seed: 42
|
| 49 |
trainer:
|
| 50 |
distributed_backend: deepspeed
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
freeze_llm_layers: []
|
| 52 |
freeze_modules: ''
|
| 53 |
freeze_tied_embedding: false
|
|
|
|
| 58 |
action_model: 0.0001
|
| 59 |
base: 2.0e-05
|
| 60 |
qwen_vl_interface: 1.0e-05
|
|
|
|
| 61 |
lr_scheduler_type: cosine_with_min_lr
|
| 62 |
max_train_steps: 4000
|
| 63 |
num_warmup_steps: 100
|
|
|
|
| 68 |
eps: 1.0e-08
|
| 69 |
fused: true
|
| 70 |
weight_decay: 1.0e-08
|
| 71 |
+
pretrained_checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
scheduler_specific_kwargs:
|
| 73 |
min_lr: 1.0e-06
|
|
|
|
|
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/hydra/.hydra/config.yaml
CHANGED
|
@@ -2,7 +2,6 @@ framework:
|
|
| 2 |
qwenvl:
|
| 3 |
base_vlm: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 4 |
attn_implementation: flash_attention_2
|
| 5 |
-
flex_backend: triton
|
| 6 |
enable_gradient_checkpointing: true
|
| 7 |
action_model:
|
| 8 |
state_dim: 7
|
|
@@ -130,8 +129,8 @@ trainer:
|
|
| 130 |
gradient_accumulation_steps: 16
|
| 131 |
distributed_backend: deepspeed
|
| 132 |
is_resume: true
|
| 133 |
-
pretrained_checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/
|
| 134 |
-
resume_step:
|
| 135 |
reload_modules: null
|
| 136 |
optimizer:
|
| 137 |
name: AdamW
|
|
|
|
| 2 |
qwenvl:
|
| 3 |
base_vlm: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 4 |
attn_implementation: flash_attention_2
|
|
|
|
| 5 |
enable_gradient_checkpointing: true
|
| 6 |
action_model:
|
| 7 |
state_dim: 7
|
|
|
|
| 129 |
gradient_accumulation_steps: 16
|
| 130 |
distributed_backend: deepspeed
|
| 131 |
is_resume: true
|
| 132 |
+
pretrained_checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 133 |
+
resume_step: 4000
|
| 134 |
reload_modules: null
|
| 135 |
optimizer:
|
| 136 |
name: AdamW
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/hydra/.hydra/hydra.yaml
CHANGED
|
@@ -118,7 +118,6 @@ hydra:
|
|
| 118 |
- mode=single
|
| 119 |
- ++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 120 |
- ++framework.qwenvl.attn_implementation=flash_attention_2
|
| 121 |
-
- ++framework.qwenvl.flex_backend=triton
|
| 122 |
- ++framework.qwenvl.enable_gradient_checkpointing=true
|
| 123 |
- ++framework.action_model.state_dim=7
|
| 124 |
- ++framework.action_model.loss_type=discrete_ce
|
|
@@ -315,8 +314,8 @@ hydra:
|
|
| 315 |
- ++version_id=0.21
|
| 316 |
- ++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 317 |
- ++trainer.is_resume=true
|
| 318 |
-
- ++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/
|
| 319 |
-
- ++trainer.resume_step=
|
| 320 |
- ++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 321 |
- ++datasets.vla_data.data_mix=flappy_train__bridge
|
| 322 |
- ++datasets.vla_data.eval_data_mix=flappy_train__bridge__val
|
|
@@ -325,7 +324,7 @@ hydra:
|
|
| 325 |
job:
|
| 326 |
name: train_starvla_hydra
|
| 327 |
chdir: false
|
| 328 |
-
override_dirname: ++auth.env_file=null,++auth.hf_token_env=HF_TOKEN,++auth.wandb_api_key_env=WANDB_API_KEY,++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct,++bridge_base_model.local_dir.gr00t=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++bridge_base_model.local_dir.openvla=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++bridge_base_model.local_dir.pi05=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++bridge_base_model.local_dir.pi0=playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action,++bridge_base_model.repo_id.gr00t=Qwen/Qwen3-VL-4B-Instruct,++bridge_base_model.repo_id.openvla=Qwen/Qwen3-VL-4B-Instruct,++bridge_base_model.repo_id.pi05=Qwen/Qwen3-VL-4B-Instruct,++bridge_base_model.repo_id.pi0=StarVLA/Qwen2.5-VL-3B-Instruct-Action,++checkpoint.hf_repo_id=null,++checkpoint.load=auto,++checkpoint.local.keep_last_n=1,++checkpoint.save_best_model=false,++checkpoint.save_final_model=true,++checkpoint.save_pt_file=false,++checkpoint.save_safetensors_file=true,++checkpoint.save_training_state=true,++checkpoint.sync.enabled=false,++checkpoint.sync.keep_last_n=0,++checkpoint.sync.repo_id=null,++checkpoint.sync.resume_policy=local_latest,++checkpoint.sync.sync_every_n_checkpoints=1,++config_yaml=null,++dataset.config_name=null,++dataset.converted_name=flappy_train,++dataset.debug_subset.enabled=false,++dataset.debug_subset.max_episodes=5,++dataset.debug_subset.suffix=debug,++dataset.episodes_per_latency=null,++dataset.force_download=false,++dataset.latency_filter=null,++dataset.max_episodes=null,++dataset.mixed_converted_name=flappy_mixed_latency_train,++dataset.mixed_latency_filter=null,++dataset.mixed_source_hf=,++dataset.setup_force=false,++dataset.single_converted_name=flappy_train,++dataset.single_latency_filter=null,++dataset.single_source_hf=,++dataset.skip_verification=false,++dataset.source_hf=,++dataset.source_subdir=null,++dataset.target_latency_unit=raw_frames,++dataset.verify_rows=200,++datasets.vla_data.action_balance.action_key=action_id,++datasets.vla_data.action_balance.enabled=false,++datasets.vla_data.action_balance.flap_id=1,++datasets.vla_data.action_balance.noop_id=0,++datasets.vla_data.action_balance.strategy=balanced_epoch,++datasets.vla_data.action_balance.target_flap_fraction=0.3,++datasets.vla_data.action_type=discrete,++datasets.vla_data.custom_mixtures_path=null,++datasets.vla_data.data_mix=flappy_train,++datasets.vla_data.data_mix=flappy_train__bridge,++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps,++datasets.vla_data.data_root_dir=playground/Datasets/rl_games,++datasets.vla_data.dataset_py=lerobot_datasets,++datasets.vla_data.eval_data_mix=flappy_train__bridge__val,++datasets.vla_data.eval_data_mix=null,++datasets.vla_data.eval_num_workers=8,++datasets.vla_data.eval_sequential_step_sampling=null,++datasets.vla_data.image_mode=single,++datasets.vla_data.include_state=true,++datasets.vla_data.latency_curriculum.computed_plan=null,++datasets.vla_data.latency_curriculum.enabled=false,++datasets.vla_data.latency_curriculum.eval_at_phase_end=false,++datasets.vla_data.latency_curriculum.final_equalization=true,++datasets.vla_data.latency_curriculum.latencies=null,++datasets.vla_data.latency_curriculum.new_latency_passes=1.0,++datasets.vla_data.latency_curriculum.phase_distributions=null,++datasets.vla_data.latency_curriculum.phase_steps=null,++datasets.vla_data.latency_curriculum.replay_passes=0.25,++datasets.vla_data.latency_curriculum.save_at_phase_end=false,++datasets.vla_data.latency_curriculum.step_budget_mode=auto,++datasets.vla_data.latency_curriculum.strategy=exclusive,++datasets.vla_data.latency_curriculum.target_total_passes=2.0,++datasets.vla_data.load_all_data_for_training=true,++datasets.vla_data.num_obs_frames=1,++datasets.vla_data.num_workers=8,++datasets.vla_data.obs_image_size=null,++datasets.vla_data.per_device_batch_size=8,++datasets.vla_data.persistent_workers=true,++datasets.vla_data.pin_memory=true,++datasets.vla_data.prefetch_factor=4,++datasets.vla_data.prompt_mode=raw,++datasets.vla_data.sequential_step_sampling=false,++datasets.vla_data.shuffle=true,++datasets.vla_data.stitch_grid=[2,2],++datasets.vla_data.video_backend=torchvision_av,++framework.action_model.action_dim=7,++framework.action_model.action_env_dim=2,++framework.action_model.action_horizon=1,++framework.action_model.future_action_window_size=0,++framework.action_model.loss_type=discrete_ce,++framework.action_model.past_action_window_size=0,++framework.action_model.state_dim=7,++framework.kv_memory.enabled=false,++framework.kv_memory.packed_train=false,++framework.kv_memory.rebased_sink=true,++framework.kv_memory.rollout_len=8,++framework.kv_memory.window=4,++framework.name=QwenOFT,++framework.qwenvl.attn_implementation=flash_attention_2,++framework.qwenvl.base_vlm=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct,++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++framework.qwenvl.enable_gradient_checkpointing=true,++
|
| 329 |
id: ???
|
| 330 |
num: ???
|
| 331 |
config_name: train
|
|
|
|
| 118 |
- mode=single
|
| 119 |
- ++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 120 |
- ++framework.qwenvl.attn_implementation=flash_attention_2
|
|
|
|
| 121 |
- ++framework.qwenvl.enable_gradient_checkpointing=true
|
| 122 |
- ++framework.action_model.state_dim=7
|
| 123 |
- ++framework.action_model.loss_type=discrete_ce
|
|
|
|
| 314 |
- ++version_id=0.21
|
| 315 |
- ++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 316 |
- ++trainer.is_resume=true
|
| 317 |
+
- ++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 318 |
+
- ++trainer.resume_step=4000
|
| 319 |
- ++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 320 |
- ++datasets.vla_data.data_mix=flappy_train__bridge
|
| 321 |
- ++datasets.vla_data.eval_data_mix=flappy_train__bridge__val
|
|
|
|
| 324 |
job:
|
| 325 |
name: train_starvla_hydra
|
| 326 |
chdir: false
|
| 327 |
+
override_dirname: ++auth.env_file=null,++auth.hf_token_env=HF_TOKEN,++auth.wandb_api_key_env=WANDB_API_KEY,++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct,++bridge_base_model.local_dir.gr00t=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++bridge_base_model.local_dir.openvla=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++bridge_base_model.local_dir.pi05=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++bridge_base_model.local_dir.pi0=playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action,++bridge_base_model.repo_id.gr00t=Qwen/Qwen3-VL-4B-Instruct,++bridge_base_model.repo_id.openvla=Qwen/Qwen3-VL-4B-Instruct,++bridge_base_model.repo_id.pi05=Qwen/Qwen3-VL-4B-Instruct,++bridge_base_model.repo_id.pi0=StarVLA/Qwen2.5-VL-3B-Instruct-Action,++checkpoint.hf_repo_id=null,++checkpoint.load=auto,++checkpoint.local.keep_last_n=1,++checkpoint.save_best_model=false,++checkpoint.save_final_model=true,++checkpoint.save_pt_file=false,++checkpoint.save_safetensors_file=true,++checkpoint.save_training_state=true,++checkpoint.sync.enabled=false,++checkpoint.sync.keep_last_n=0,++checkpoint.sync.repo_id=null,++checkpoint.sync.resume_policy=local_latest,++checkpoint.sync.sync_every_n_checkpoints=1,++config_yaml=null,++dataset.config_name=null,++dataset.converted_name=flappy_train,++dataset.debug_subset.enabled=false,++dataset.debug_subset.max_episodes=5,++dataset.debug_subset.suffix=debug,++dataset.episodes_per_latency=null,++dataset.force_download=false,++dataset.latency_filter=null,++dataset.max_episodes=null,++dataset.mixed_converted_name=flappy_mixed_latency_train,++dataset.mixed_latency_filter=null,++dataset.mixed_source_hf=,++dataset.setup_force=false,++dataset.single_converted_name=flappy_train,++dataset.single_latency_filter=null,++dataset.single_source_hf=,++dataset.skip_verification=false,++dataset.source_hf=,++dataset.source_subdir=null,++dataset.target_latency_unit=raw_frames,++dataset.verify_rows=200,++datasets.vla_data.action_balance.action_key=action_id,++datasets.vla_data.action_balance.enabled=false,++datasets.vla_data.action_balance.flap_id=1,++datasets.vla_data.action_balance.noop_id=0,++datasets.vla_data.action_balance.strategy=balanced_epoch,++datasets.vla_data.action_balance.target_flap_fraction=0.3,++datasets.vla_data.action_type=discrete,++datasets.vla_data.custom_mixtures_path=null,++datasets.vla_data.data_mix=flappy_train,++datasets.vla_data.data_mix=flappy_train__bridge,++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps,++datasets.vla_data.data_root_dir=playground/Datasets/rl_games,++datasets.vla_data.dataset_py=lerobot_datasets,++datasets.vla_data.eval_data_mix=flappy_train__bridge__val,++datasets.vla_data.eval_data_mix=null,++datasets.vla_data.eval_num_workers=8,++datasets.vla_data.eval_sequential_step_sampling=null,++datasets.vla_data.image_mode=single,++datasets.vla_data.include_state=true,++datasets.vla_data.latency_curriculum.computed_plan=null,++datasets.vla_data.latency_curriculum.enabled=false,++datasets.vla_data.latency_curriculum.eval_at_phase_end=false,++datasets.vla_data.latency_curriculum.final_equalization=true,++datasets.vla_data.latency_curriculum.latencies=null,++datasets.vla_data.latency_curriculum.new_latency_passes=1.0,++datasets.vla_data.latency_curriculum.phase_distributions=null,++datasets.vla_data.latency_curriculum.phase_steps=null,++datasets.vla_data.latency_curriculum.replay_passes=0.25,++datasets.vla_data.latency_curriculum.save_at_phase_end=false,++datasets.vla_data.latency_curriculum.step_budget_mode=auto,++datasets.vla_data.latency_curriculum.strategy=exclusive,++datasets.vla_data.latency_curriculum.target_total_passes=2.0,++datasets.vla_data.load_all_data_for_training=true,++datasets.vla_data.num_obs_frames=1,++datasets.vla_data.num_workers=8,++datasets.vla_data.obs_image_size=null,++datasets.vla_data.per_device_batch_size=8,++datasets.vla_data.persistent_workers=true,++datasets.vla_data.pin_memory=true,++datasets.vla_data.prefetch_factor=4,++datasets.vla_data.prompt_mode=raw,++datasets.vla_data.sequential_step_sampling=false,++datasets.vla_data.shuffle=true,++datasets.vla_data.stitch_grid=[2,2],++datasets.vla_data.video_backend=torchvision_av,++framework.action_model.action_dim=7,++framework.action_model.action_env_dim=2,++framework.action_model.action_horizon=1,++framework.action_model.future_action_window_size=0,++framework.action_model.loss_type=discrete_ce,++framework.action_model.past_action_window_size=0,++framework.action_model.state_dim=7,++framework.kv_memory.enabled=false,++framework.kv_memory.packed_train=false,++framework.kv_memory.rebased_sink=true,++framework.kv_memory.rollout_len=8,++framework.kv_memory.window=4,++framework.name=QwenOFT,++framework.qwenvl.attn_implementation=flash_attention_2,++framework.qwenvl.base_vlm=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct,++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++framework.qwenvl.enable_gradient_checkpointing=true,++initialization.checkpoint_filename=checkpoints/steps_5000_pytorch_model.pt,++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1,++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1,++is_debug=false,++output_dir=null,++paths.accelerate_config=starVLA/config/deepseeds/deepspeed_zero2.yaml,++paths.base_model_dir=playground/Pretrained_models/Qwen3-VL-4B-Instruct,++paths.dataset_cache_dir=null,++paths.dataset_local_dir=data/memory/flappy_fix_latency_3_200ep_7k2steps,++paths.run_root_dir=results/Checkpoints,++rl_games.action_carrier=bridge,++rl_games.env_eval.action_chunk_execution.chunk_size=null,++rl_games.env_eval.action_chunk_execution.enabled=false,++rl_games.env_eval.distributed_mode=rank_sharded,++rl_games.env_eval.enabled=true,++rl_games.env_eval.eval_backend=latency_bench,++rl_games.env_eval.eval_parallel_envs=5,++rl_games.env_eval.fixed_episode_seeds=true,++rl_games.env_eval.frameskip=1,++rl_games.env_eval.ghost_trail.gamma=1.3,++rl_games.env_eval.ghost_trail.ground_fraction=0.22,++rl_games.env_eval.ghost_trail.history_frames=5,++rl_games.env_eval.ghost_trail.min_alpha=35,++rl_games.env_eval.ghost_trail.scroll_px_per_step=4.0,++rl_games.env_eval.image_size=224,++rl_games.env_eval.image_transform=raw_rgb,++rl_games.env_eval.latency.mode=single,++rl_games.env_eval.latency.prompt_map_path=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps/flappy_train__bridge/latency_prompt_map.json,++rl_games.env_eval.latency.prompt_map_path=null,++rl_games.env_eval.latency.values=[0],++rl_games.env_eval.latency_seed_stride=0,++rl_games.env_eval.mid_train.enabled=true,++rl_games.env_eval.mid_train.interval_steps=250,++rl_games.env_eval.mid_train.latencies=[3],++rl_games.env_eval.mid_train.max_steps_per_episode=3600,++rl_games.env_eval.mid_train.num_episodes=20,++rl_games.env_eval.post_train.enabled=true,++rl_games.env_eval.post_train.latencies=[3],++rl_games.env_eval.post_train.max_steps_per_episode=3600,++rl_games.env_eval.post_train.num_episodes=50,++rl_games.env_eval.prompt_mode=raw,++rl_games.env_eval.seed=42,++rl_games.env_eval.task_description=,++rl_games.env_eval.task_seed_stride=0,++rl_games.env_eval.vectorized.batch_size=1,++rl_games.env_eval.vectorized.enabled=false,++rl_games.initialization_mode=bridge,++rl_games.model_alias=openvla,++rl_games.task=flappy,++run_id=flappy_fix_latency_3_200ep_7k2steps_single_baseline,++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints,++run_root_dir=results/Checkpoints,++seed=42,++trainer.cc_f1_tolerance=1,++trainer.distributed_backend=deepspeed,++trainer.eval_action_classification=false,++trainer.eval_action_classification_interval=null,++trainer.eval_interval=250,++trainer.eval_num_batches=50,++trainer.freeze_llm_layers=[],++trainer.freeze_modules=,++trainer.freeze_tied_embedding=false,++trainer.freeze_vit=false,++trainer.gradient_accumulation_steps=16,++trainer.gradient_clipping=1.0,++trainer.is_resume=false,++trainer.is_resume=true,++trainer.learning_rate.action_model=0.0001,++trainer.learning_rate.base=2e-05,++trainer.learning_rate.qwen_vl_interface=1e-05,++trainer.logging_frequency=1,++trainer.loss_scale.vla=1.0,++trainer.loss_scale.vlm=0.1,++trainer.lr_scheduler_type=cosine_with_min_lr,++trainer.max_grad_norm=1.0,++trainer.max_train_steps=4000,++trainer.num_warmup_steps=100,++trainer.optimizer.betas=[0.9,0.95],++trainer.optimizer.eps=1e-08,++trainer.optimizer.fused=true,++trainer.optimizer.name=AdamW,++trainer.optimizer.weight_decay=1e-08,++trainer.per_latency_eval_num_batches=null,++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state,++trainer.pretrained_checkpoint=null,++trainer.profile_timing.enabled=false,++trainer.profile_timing.log_interval=10,++trainer.reload_modules=null,++trainer.resume_step=0,++trainer.resume_step=4000,++trainer.save_format=pt,++trainer.save_interval=500,++trainer.scheduler_specific_kwargs.min_lr=1e-06,++trainer.weight_decay=0.0,++version_id=0.21,++workspace_dir=WORKSPACE_DIR,env=flappy,init=bridge,mode=single,model=openvla
|
| 328 |
id: ???
|
| 329 |
num: ???
|
| 330 |
config_name: train
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/hydra/.hydra/overrides.yaml
CHANGED
|
@@ -4,7 +4,6 @@
|
|
| 4 |
- mode=single
|
| 5 |
- ++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 6 |
- ++framework.qwenvl.attn_implementation=flash_attention_2
|
| 7 |
-
- ++framework.qwenvl.flex_backend=triton
|
| 8 |
- ++framework.qwenvl.enable_gradient_checkpointing=true
|
| 9 |
- ++framework.action_model.state_dim=7
|
| 10 |
- ++framework.action_model.loss_type=discrete_ce
|
|
@@ -201,8 +200,8 @@
|
|
| 201 |
- ++version_id=0.21
|
| 202 |
- ++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 203 |
- ++trainer.is_resume=true
|
| 204 |
-
- ++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/
|
| 205 |
-
- ++trainer.resume_step=
|
| 206 |
- ++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 207 |
- ++datasets.vla_data.data_mix=flappy_train__bridge
|
| 208 |
- ++datasets.vla_data.eval_data_mix=flappy_train__bridge__val
|
|
|
|
| 4 |
- mode=single
|
| 5 |
- ++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 6 |
- ++framework.qwenvl.attn_implementation=flash_attention_2
|
|
|
|
| 7 |
- ++framework.qwenvl.enable_gradient_checkpointing=true
|
| 8 |
- ++framework.action_model.state_dim=7
|
| 9 |
- ++framework.action_model.loss_type=discrete_ce
|
|
|
|
| 200 |
- ++version_id=0.21
|
| 201 |
- ++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 202 |
- ++trainer.is_resume=true
|
| 203 |
+
- ++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 204 |
+
- ++trainer.resume_step=4000
|
| 205 |
- ++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 206 |
- ++datasets.vla_data.data_mix=flappy_train__bridge
|
| 207 |
- ++datasets.vla_data.eval_data_mix=flappy_train__bridge__val
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/hydra/train_starvla_hydra.log
CHANGED
|
@@ -4803,3 +4803,315 @@ Cuda failure 'out of memory'
|
|
| 4803 |
[2026-07-26 16:10:28,183][starVLA.training.train_starvla][INFO] - ... and that's all, folks!
|
| 4804 |
[2026-07-26 16:10:32,480][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 4805 |
[2026-07-26 16:10:32,481][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4803 |
[2026-07-26 16:10:28,183][starVLA.training.train_starvla][INFO] - ... and that's all, folks!
|
| 4804 |
[2026-07-26 16:10:32,480][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 4805 |
[2026-07-26 16:10:32,481][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 4806 |
+
[2026-07-27 21:40:23,063][starVLA.training.train_starvla][INFO] - VLA Training :: Warming Up
|
| 4807 |
+
[2026-07-27 21:40:23,064][starVLA.model.framework.share_tools][INFO] - [*] [apply_config_compat] normalised config from version_id=0.21 to '0.21'
|
| 4808 |
+
[2026-07-27 21:40:23,064][starVLA.training.train_starvla][INFO] - VLA Training :: Warming Up
|
| 4809 |
+
[2026-07-27 21:40:23,325][huggingface_hub._login][WARNING] - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
|
| 4810 |
+
[2026-07-27 21:40:23,325][starVLA.training.rl_games.auth][INFO] - Logged into Hugging Face Hub
|
| 4811 |
+
[2026-07-27 21:40:23,341][huggingface_hub._login][WARNING] - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
|
| 4812 |
+
[2026-07-27 21:40:23,341][starVLA.training.rl_games.auth][INFO] - Logged into Hugging Face Hub
|
| 4813 |
+
[2026-07-27 21:40:23,766][starVLA.training.rl_games.auth][INFO] - Logged into Weights & Biases
|
| 4814 |
+
[2026-07-27 21:40:23,781][starVLA.training.train_starvla][INFO] - ✅ Configuration wrapped for access tracking
|
| 4815 |
+
[2026-07-27 21:40:23,788][starVLA.training.rl_games.auth][INFO] - Logged into Weights & Biases
|
| 4816 |
+
[2026-07-27 21:40:23,803][starVLA.training.train_starvla][INFO] - ✅ Configuration wrapped for access tracking
|
| 4817 |
+
[2026-07-27 21:40:26,545][starVLA.training.train_starvla][INFO] - Creating VLA Dataset with Mixture `flappy_train__bridge`
|
| 4818 |
+
[2026-07-27 21:40:26,549][starVLA.training.train_starvla][INFO] - Creating VLA Dataset with Mixture `flappy_train__bridge`
|
| 4819 |
+
[2026-07-27 21:40:27,527][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Behavior (data_config): ['BEHAVIOR_challenge']
|
| 4820 |
+
[2026-07-27 21:40:27,528][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from DOMINO: ['robotwin']
|
| 4821 |
+
[2026-07-27 21:40:27,528][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from DOMINO (data_config): []
|
| 4822 |
+
[2026-07-27 21:40:27,528][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from DOMINO (data_config): ['domino', 'domino_clean', 'domino_random', 'domino_cotrain']
|
| 4823 |
+
[2026-07-27 21:40:27,529][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Franka: ['custom_robot_config', 'demo_sim_franka_delta_joints', 'SO101']
|
| 4824 |
+
[2026-07-27 21:40:27,529][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Franka (data_config): []
|
| 4825 |
+
[2026-07-27 21:40:27,529][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Franka (data_config): ['custom_dataset', 'custom_dataset_2', 'demo_sim_pick_place', 'SO101_pick']
|
| 4826 |
+
[2026-07-27 21:40:27,530][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from LIBERO: ['libero_franka']
|
| 4827 |
+
[2026-07-27 21:40:27,530][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from LIBERO (data_config): []
|
| 4828 |
+
[2026-07-27 21:40:27,530][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from LIBERO (data_config): ['libero_all', 'libero_goal', 'multi_robot']
|
| 4829 |
+
[2026-07-27 21:40:27,531][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from RoboChallenge_table30v2: ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 4830 |
+
[2026-07-27 21:40:27,531][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from RoboChallenge_table30v2 (data_config): ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 4831 |
+
[2026-07-27 21:40:27,531][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from RoboChallenge_table30v2 (data_config): ['robochallenge_table30v2_shred_paper', 'robochallenge_table30v2_ur5_all', 'robochallenge_table30v2_arx5_all', 'robochallenge_table30v2_dosw1_all']
|
| 4832 |
+
[2026-07-27 21:40:27,531][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Behavior (data_config): ['BEHAVIOR_challenge']
|
| 4833 |
+
[2026-07-27 21:40:27,532][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from DOMINO: ['robotwin']
|
| 4834 |
+
[2026-07-27 21:40:27,532][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from DOMINO (data_config): []
|
| 4835 |
+
[2026-07-27 21:40:27,532][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from DOMINO (data_config): ['domino', 'domino_clean', 'domino_random', 'domino_cotrain']
|
| 4836 |
+
[2026-07-27 21:40:27,533][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Franka: ['custom_robot_config', 'demo_sim_franka_delta_joints', 'SO101']
|
| 4837 |
+
[2026-07-27 21:40:27,533][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Franka (data_config): []
|
| 4838 |
+
[2026-07-27 21:40:27,533][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Franka (data_config): ['custom_dataset', 'custom_dataset_2', 'demo_sim_pick_place', 'SO101_pick']
|
| 4839 |
+
[2026-07-27 21:40:27,534][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from LIBERO: ['libero_franka']
|
| 4840 |
+
[2026-07-27 21:40:27,534][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from LIBERO (data_config): []
|
| 4841 |
+
[2026-07-27 21:40:27,534][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from LIBERO (data_config): ['libero_all', 'libero_goal', 'multi_robot']
|
| 4842 |
+
[2026-07-27 21:40:27,534][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from RoboChallenge_table30v2: ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 4843 |
+
[2026-07-27 21:40:27,534][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from RoboChallenge_table30v2 (data_config): ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 4844 |
+
[2026-07-27 21:40:27,534][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from RoboChallenge_table30v2 (data_config): ['robochallenge_table30v2_shred_paper', 'robochallenge_table30v2_ur5_all', 'robochallenge_table30v2_arx5_all', 'robochallenge_table30v2_dosw1_all']
|
| 4845 |
+
[2026-07-27 21:40:27,532][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_365: ['panda_omron_robocasa365']
|
| 4846 |
+
[2026-07-27 21:40:27,725][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_365 (data_config): []
|
| 4847 |
+
[2026-07-27 21:40:27,725][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_365 (data_config): ['robocasa365_open_drawer_target_human', 'robocasa365_atomic_target_human_all', 'robocasa365_composite_target_human_all', 'robocasa365_target_human_all']
|
| 4848 |
+
[2026-07-27 21:40:27,726][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_tabletop: ['fourier_gr1_arms_waist']
|
| 4849 |
+
[2026-07-27 21:40:27,726][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_tabletop (data_config): []
|
| 4850 |
+
[2026-07-27 21:40:27,726][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_tabletop (data_config): ['fourier_gr1_unified_1000']
|
| 4851 |
+
[2026-07-27 21:40:27,727][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robotwin: ['robotwin', 'robotwin50', 'arx_x5']
|
| 4852 |
+
[2026-07-27 21:40:27,727][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robotwin (data_config): []
|
| 4853 |
+
[2026-07-27 21:40:27,727][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robotwin (data_config): ['robotwin_all', 'robotwin_all_50', 'robotwin', 'robotwin_task1', 'robotwin_task2', 'arx_x5']
|
| 4854 |
+
[2026-07-27 21:40:27,728][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from SimplerEnv: ['oxe_droid', 'oxe_bridge', 'oxe_rt1']
|
| 4855 |
+
[2026-07-27 21:40:27,728][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from SimplerEnv (data_config): []
|
| 4856 |
+
[2026-07-27 21:40:27,728][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from SimplerEnv (data_config): ['bridge', 'bridge_rt_1']
|
| 4857 |
+
[2026-07-27 21:40:27,728][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from VLA-Arena: ['vla_arena_franka']
|
| 4858 |
+
[2026-07-27 21:40:27,728][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from VLA-Arena (data_config): []
|
| 4859 |
+
[2026-07-27 21:40:27,728][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from VLA-Arena (data_config): ['vla_arena_L0_S', 'vla_arena_L0_M', 'vla_arena_L0_L']
|
| 4860 |
+
[2026-07-27 21:40:27,729][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from rl_games: ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 4861 |
+
[2026-07-27 21:40:27,729][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from rl_games (data_config): ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 4862 |
+
[2026-07-27 21:40:27,729][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from rl_games (data_config): ['flappy_train', 'flappy_train__bridge', 'flappy_mixed_latency_train', 'flappy_mixed_latency_train__bridge', 'demon_attack_train', 'demon_attack_train__bridge', 'demon_attack_mixed_latency_train', 'demon_attack_mixed_latency_train__bridge', 'deadly_corridor_train', 'deadly_corridor_train__bridge', 'deadly_corridor_mixed_latency_train', 'deadly_corridor_mixed_latency_train__bridge']
|
| 4863 |
+
[2026-07-27 21:40:27,535][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_365: ['panda_omron_robocasa365']
|
| 4864 |
+
[2026-07-27 21:40:27,732][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_365 (data_config): []
|
| 4865 |
+
[2026-07-27 21:40:27,732][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_365 (data_config): ['robocasa365_open_drawer_target_human', 'robocasa365_atomic_target_human_all', 'robocasa365_composite_target_human_all', 'robocasa365_target_human_all']
|
| 4866 |
+
[2026-07-27 21:40:27,733][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_tabletop: ['fourier_gr1_arms_waist']
|
| 4867 |
+
[2026-07-27 21:40:27,733][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_tabletop (data_config): []
|
| 4868 |
+
[2026-07-27 21:40:27,733][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_tabletop (data_config): ['fourier_gr1_unified_1000']
|
| 4869 |
+
[2026-07-27 21:40:27,741][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robotwin: ['robotwin', 'robotwin50', 'arx_x5']
|
| 4870 |
+
[2026-07-27 21:40:27,742][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robotwin (data_config): []
|
| 4871 |
+
[2026-07-27 21:40:27,742][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robotwin (data_config): ['robotwin_all', 'robotwin_all_50', 'robotwin', 'robotwin_task1', 'robotwin_task2', 'arx_x5']
|
| 4872 |
+
[2026-07-27 21:40:27,742][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from SimplerEnv: ['oxe_droid', 'oxe_bridge', 'oxe_rt1']
|
| 4873 |
+
[2026-07-27 21:40:27,742][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from SimplerEnv (data_config): []
|
| 4874 |
+
[2026-07-27 21:40:27,742][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from SimplerEnv (data_config): ['bridge', 'bridge_rt_1']
|
| 4875 |
+
[2026-07-27 21:40:27,743][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from VLA-Arena: ['vla_arena_franka']
|
| 4876 |
+
[2026-07-27 21:40:27,743][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from VLA-Arena (data_config): []
|
| 4877 |
+
[2026-07-27 21:40:27,743][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from VLA-Arena (data_config): ['vla_arena_L0_S', 'vla_arena_L0_M', 'vla_arena_L0_L']
|
| 4878 |
+
[2026-07-27 21:40:27,744][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from rl_games: ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 4879 |
+
[2026-07-27 21:40:27,744][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from rl_games (data_config): ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 4880 |
+
[2026-07-27 21:40:27,744][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from rl_games (data_config): ['flappy_train', 'flappy_train__bridge', 'flappy_mixed_latency_train', 'flappy_mixed_latency_train__bridge', 'demon_attack_train', 'demon_attack_train__bridge', 'demon_attack_mixed_latency_train', 'demon_attack_mixed_latency_train__bridge', 'deadly_corridor_train', 'deadly_corridor_train__bridge', 'deadly_corridor_mixed_latency_train', 'deadly_corridor_mixed_latency_train__bridge']
|
| 4881 |
+
[2026-07-27 21:40:28,943][starVLA.dataloader][INFO] - Created train VLA dataloader with num_workers=8, persistent_workers=True, cpu_only_workers=True
|
| 4882 |
+
[2026-07-27 21:40:28,945][starVLA.training.train_starvla][INFO] - Creating VLA Eval Dataset with Mixture `flappy_train__bridge__val`
|
| 4883 |
+
[2026-07-27 21:40:28,947][starVLA.training.train_starvla][INFO] - Creating VLA Eval Dataset with Mixture `flappy_train__bridge__val`
|
| 4884 |
+
[2026-07-27 21:40:28,963][starVLA.dataloader][INFO] - Created eval VLA dataloader with num_workers=8, persistent_workers=True, cpu_only_workers=True
|
| 4885 |
+
[2026-07-27 21:40:28,971][starVLA.training.train_starvla][INFO] - LR Group qwen_vl_interface: lr=1e-05, num_params=713
|
| 4886 |
+
[2026-07-27 21:40:28,971][starVLA.training.train_starvla][INFO] - LR Group action_model: lr=0.0001, num_params=16
|
| 4887 |
+
[2026-07-27 21:40:30,364][starVLA.training.train_starvla][INFO] - Will resume full training state from checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state, steps: 4000
|
| 4888 |
+
[2026-07-27 21:40:30,375][starVLA.training.train_starvla][INFO] - 📝 Full config saved at /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.full.yaml
|
| 4889 |
+
[2026-07-27 21:40:30,387][starVLA.training.train_starvla][INFO] - 📊 Accessed config snapshot saved at /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.yaml
|
| 4890 |
+
[2026-07-27 21:40:30,391][starVLA.training.train_starvla][INFO] - Will resume full training state from checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state, steps: 4000
|
| 4891 |
+
[2026-07-27 21:40:30,395][accelerate.accelerator][INFO] - Since you passed both train and evaluation dataloader, `is_train_batch_min` (here True will decide the `train_batch_size` (8).
|
| 4892 |
+
[2026-07-27 21:41:06,756][accelerate.accelerator][INFO] - Loading states from /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 4893 |
+
[2026-07-27 21:41:06,756][accelerate.accelerator][INFO] - Loading DeepSpeed Model and Optimizer
|
| 4894 |
+
[2026-07-27 21:41:39,175][accelerate.accelerator][INFO] - DeepSpeed Model and Optimizer loaded from input dir /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state/pytorch_model
|
| 4895 |
+
[2026-07-27 21:41:39,175][accelerate.checkpointing][INFO] - All model weights loaded successfully
|
| 4896 |
+
[2026-07-27 21:41:39,175][accelerate.checkpointing][INFO] - All optimizer states loaded successfully
|
| 4897 |
+
[2026-07-27 21:41:39,175][accelerate.checkpointing][INFO] - All scheduler states loaded successfully
|
| 4898 |
+
[2026-07-27 21:41:39,175][accelerate.checkpointing][INFO] - All dataloader sampler states loaded successfully
|
| 4899 |
+
[2026-07-27 21:41:39,182][accelerate.checkpointing][INFO] - All random states loaded successfully
|
| 4900 |
+
[2026-07-27 21:41:39,183][accelerate.accelerator][INFO] - Loading in 0 custom states
|
| 4901 |
+
[2026-07-27 21:41:42,198][starVLA.training.train_starvla][INFO] - ***** Training Configuration *****
|
| 4902 |
+
[2026-07-27 21:41:42,199][starVLA.training.train_starvla][INFO] - Total optimization steps = 4000
|
| 4903 |
+
[2026-07-27 21:41:42,199][starVLA.training.train_starvla][INFO] - Per device batch size = 8
|
| 4904 |
+
[2026-07-27 21:41:42,199][starVLA.training.train_starvla][INFO] - Gradient accumulation steps = 16
|
| 4905 |
+
[2026-07-27 21:41:42,199][starVLA.training.train_starvla][INFO] - Total batch size = 256
|
| 4906 |
+
[2026-07-27 21:41:42,200][starVLA.training.train_starvla][INFO] - Training complete. Final training state is saved under /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints
|
| 4907 |
+
[2026-07-27 21:41:42,883][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 4908 |
+
[2026-07-27 21:41:42,889][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 4909 |
+
[2026-07-27 23:10:19,778][starVLA.model.framework.share_tools][INFO] - [*] [apply_config_compat] normalised config from version_id=0.21 to '0.21'
|
| 4910 |
+
[2026-07-27 23:10:19,779][starVLA.training.train_starvla][INFO] - VLA Training :: Warming Up
|
| 4911 |
+
[2026-07-27 23:10:19,779][starVLA.training.train_starvla][INFO] - VLA Training :: Warming Up
|
| 4912 |
+
[2026-07-27 23:10:20,043][huggingface_hub._login][WARNING] - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
|
| 4913 |
+
[2026-07-27 23:10:20,043][starVLA.training.rl_games.auth][INFO] - Logged into Hugging Face Hub
|
| 4914 |
+
[2026-07-27 23:10:20,047][huggingface_hub._login][WARNING] - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
|
| 4915 |
+
[2026-07-27 23:10:20,047][starVLA.training.rl_games.auth][INFO] - Logged into Hugging Face Hub
|
| 4916 |
+
[2026-07-27 23:10:20,579][starVLA.training.rl_games.auth][INFO] - Logged into Weights & Biases
|
| 4917 |
+
[2026-07-27 23:10:20,594][starVLA.training.train_starvla][INFO] - ✅ Configuration wrapped for access tracking
|
| 4918 |
+
[2026-07-27 23:10:20,767][starVLA.training.rl_games.auth][INFO] - Logged into Weights & Biases
|
| 4919 |
+
[2026-07-27 23:10:20,782][starVLA.training.train_starvla][INFO] - ✅ Configuration wrapped for access tracking
|
| 4920 |
+
[2026-07-27 23:10:23,262][starVLA.training.train_starvla][INFO] - Creating VLA Dataset with Mixture `flappy_train__bridge`
|
| 4921 |
+
[2026-07-27 23:10:23,277][starVLA.training.train_starvla][INFO] - Creating VLA Dataset with Mixture `flappy_train__bridge`
|
| 4922 |
+
[2026-07-27 23:10:24,245][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Behavior (data_config): ['BEHAVIOR_challenge']
|
| 4923 |
+
[2026-07-27 23:10:24,246][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from DOMINO: ['robotwin']
|
| 4924 |
+
[2026-07-27 23:10:24,246][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from DOMINO (data_config): []
|
| 4925 |
+
[2026-07-27 23:10:24,246][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from DOMINO (data_config): ['domino', 'domino_clean', 'domino_random', 'domino_cotrain']
|
| 4926 |
+
[2026-07-27 23:10:24,247][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Franka: ['custom_robot_config', 'demo_sim_franka_delta_joints', 'SO101']
|
| 4927 |
+
[2026-07-27 23:10:24,247][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Franka (data_config): []
|
| 4928 |
+
[2026-07-27 23:10:24,247][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Franka (data_config): ['custom_dataset', 'custom_dataset_2', 'demo_sim_pick_place', 'SO101_pick']
|
| 4929 |
+
[2026-07-27 23:10:24,248][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from LIBERO: ['libero_franka']
|
| 4930 |
+
[2026-07-27 23:10:24,248][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from LIBERO (data_config): []
|
| 4931 |
+
[2026-07-27 23:10:24,248][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from LIBERO (data_config): ['libero_all', 'libero_goal', 'multi_robot']
|
| 4932 |
+
[2026-07-27 23:10:24,248][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from RoboChallenge_table30v2: ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 4933 |
+
[2026-07-27 23:10:24,249][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from RoboChallenge_table30v2 (data_config): ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 4934 |
+
[2026-07-27 23:10:24,249][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from RoboChallenge_table30v2 (data_config): ['robochallenge_table30v2_shred_paper', 'robochallenge_table30v2_ur5_all', 'robochallenge_table30v2_arx5_all', 'robochallenge_table30v2_dosw1_all']
|
| 4935 |
+
[2026-07-27 23:10:24,250][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_365: ['panda_omron_robocasa365']
|
| 4936 |
+
[2026-07-27 23:10:24,250][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_365 (data_config): []
|
| 4937 |
+
[2026-07-27 23:10:24,250][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_365 (data_config): ['robocasa365_open_drawer_target_human', 'robocasa365_atomic_target_human_all', 'robocasa365_composite_target_human_all', 'robocasa365_target_human_all']
|
| 4938 |
+
[2026-07-27 23:10:24,251][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_tabletop: ['fourier_gr1_arms_waist']
|
| 4939 |
+
[2026-07-27 23:10:24,251][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_tabletop (data_config): []
|
| 4940 |
+
[2026-07-27 23:10:24,251][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_tabletop (data_config): ['fourier_gr1_unified_1000']
|
| 4941 |
+
[2026-07-27 23:10:24,252][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robotwin: ['robotwin', 'robotwin50', 'arx_x5']
|
| 4942 |
+
[2026-07-27 23:10:24,252][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robotwin (data_config): []
|
| 4943 |
+
[2026-07-27 23:10:24,252][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robotwin (data_config): ['robotwin_all', 'robotwin_all_50', 'robotwin', 'robotwin_task1', 'robotwin_task2', 'arx_x5']
|
| 4944 |
+
[2026-07-27 23:10:24,252][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from SimplerEnv: ['oxe_droid', 'oxe_bridge', 'oxe_rt1']
|
| 4945 |
+
[2026-07-27 23:10:24,253][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from SimplerEnv (data_config): []
|
| 4946 |
+
[2026-07-27 23:10:24,253][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from SimplerEnv (data_config): ['bridge', 'bridge_rt_1']
|
| 4947 |
+
[2026-07-27 23:10:24,253][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from VLA-Arena: ['vla_arena_franka']
|
| 4948 |
+
[2026-07-27 23:10:24,253][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from VLA-Arena (data_config): []
|
| 4949 |
+
[2026-07-27 23:10:24,253][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from VLA-Arena (data_config): ['vla_arena_L0_S', 'vla_arena_L0_M', 'vla_arena_L0_L']
|
| 4950 |
+
[2026-07-27 23:10:24,254][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from rl_games: ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 4951 |
+
[2026-07-27 23:10:24,254][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from rl_games (data_config): ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 4952 |
+
[2026-07-27 23:10:24,254][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from rl_games (data_config): ['flappy_train', 'flappy_train__bridge', 'flappy_mixed_latency_train', 'flappy_mixed_latency_train__bridge', 'demon_attack_train', 'demon_attack_train__bridge', 'demon_attack_mixed_latency_train', 'demon_attack_mixed_latency_train__bridge', 'deadly_corridor_train', 'deadly_corridor_train__bridge', 'deadly_corridor_mixed_latency_train', 'deadly_corridor_mixed_latency_train__bridge']
|
| 4953 |
+
[2026-07-27 23:10:24,302][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Behavior (data_config): ['BEHAVIOR_challenge']
|
| 4954 |
+
[2026-07-27 23:10:24,303][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from DOMINO: ['robotwin']
|
| 4955 |
+
[2026-07-27 23:10:24,303][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from DOMINO (data_config): []
|
| 4956 |
+
[2026-07-27 23:10:24,303][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from DOMINO (data_config): ['domino', 'domino_clean', 'domino_random', 'domino_cotrain']
|
| 4957 |
+
[2026-07-27 23:10:24,304][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Franka: ['custom_robot_config', 'demo_sim_franka_delta_joints', 'SO101']
|
| 4958 |
+
[2026-07-27 23:10:24,304][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Franka (data_config): []
|
| 4959 |
+
[2026-07-27 23:10:24,304][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Franka (data_config): ['custom_dataset', 'custom_dataset_2', 'demo_sim_pick_place', 'SO101_pick']
|
| 4960 |
+
[2026-07-27 23:10:24,305][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from LIBERO: ['libero_franka']
|
| 4961 |
+
[2026-07-27 23:10:24,305][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from LIBERO (data_config): []
|
| 4962 |
+
[2026-07-27 23:10:24,305][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from LIBERO (data_config): ['libero_all', 'libero_goal', 'multi_robot']
|
| 4963 |
+
[2026-07-27 23:10:24,306][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from RoboChallenge_table30v2: ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 4964 |
+
[2026-07-27 23:10:24,306][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from RoboChallenge_table30v2 (data_config): ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 4965 |
+
[2026-07-27 23:10:24,306][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from RoboChallenge_table30v2 (data_config): ['robochallenge_table30v2_shred_paper', 'robochallenge_table30v2_ur5_all', 'robochallenge_table30v2_arx5_all', 'robochallenge_table30v2_dosw1_all']
|
| 4966 |
+
[2026-07-27 23:10:24,307][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_365: ['panda_omron_robocasa365']
|
| 4967 |
+
[2026-07-27 23:10:24,307][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_365 (data_config): []
|
| 4968 |
+
[2026-07-27 23:10:24,307][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_365 (data_config): ['robocasa365_open_drawer_target_human', 'robocasa365_atomic_target_human_all', 'robocasa365_composite_target_human_all', 'robocasa365_target_human_all']
|
| 4969 |
+
[2026-07-27 23:10:24,308][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_tabletop: ['fourier_gr1_arms_waist']
|
| 4970 |
+
[2026-07-27 23:10:24,308][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_tabletop (data_config): []
|
| 4971 |
+
[2026-07-27 23:10:24,308][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_tabletop (data_config): ['fourier_gr1_unified_1000']
|
| 4972 |
+
[2026-07-27 23:10:24,309][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robotwin: ['robotwin', 'robotwin50', 'arx_x5']
|
| 4973 |
+
[2026-07-27 23:10:24,309][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robotwin (data_config): []
|
| 4974 |
+
[2026-07-27 23:10:24,309][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robotwin (data_config): ['robotwin_all', 'robotwin_all_50', 'robotwin', 'robotwin_task1', 'robotwin_task2', 'arx_x5']
|
| 4975 |
+
[2026-07-27 23:10:24,309][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from SimplerEnv: ['oxe_droid', 'oxe_bridge', 'oxe_rt1']
|
| 4976 |
+
[2026-07-27 23:10:24,310][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from SimplerEnv (data_config): []
|
| 4977 |
+
[2026-07-27 23:10:24,310][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from SimplerEnv (data_config): ['bridge', 'bridge_rt_1']
|
| 4978 |
+
[2026-07-27 23:10:24,310][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from VLA-Arena: ['vla_arena_franka']
|
| 4979 |
+
[2026-07-27 23:10:24,310][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from VLA-Arena (data_config): []
|
| 4980 |
+
[2026-07-27 23:10:24,310][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from VLA-Arena (data_config): ['vla_arena_L0_S', 'vla_arena_L0_M', 'vla_arena_L0_L']
|
| 4981 |
+
[2026-07-27 23:10:24,311][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from rl_games: ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 4982 |
+
[2026-07-27 23:10:24,311][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from rl_games (data_config): ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 4983 |
+
[2026-07-27 23:10:24,311][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from rl_games (data_config): ['flappy_train', 'flappy_train__bridge', 'flappy_mixed_latency_train', 'flappy_mixed_latency_train__bridge', 'demon_attack_train', 'demon_attack_train__bridge', 'demon_attack_mixed_latency_train', 'demon_attack_mixed_latency_train__bridge', 'deadly_corridor_train', 'deadly_corridor_train__bridge', 'deadly_corridor_mixed_latency_train', 'deadly_corridor_mixed_latency_train__bridge']
|
| 4984 |
+
[2026-07-27 23:10:26,052][starVLA.training.train_starvla][INFO] - Creating VLA Eval Dataset with Mixture `flappy_train__bridge__val`
|
| 4985 |
+
[2026-07-27 23:10:26,056][starVLA.dataloader][INFO] - Created train VLA dataloader with num_workers=8, persistent_workers=True, cpu_only_workers=True
|
| 4986 |
+
[2026-07-27 23:10:26,059][starVLA.training.train_starvla][INFO] - Creating VLA Eval Dataset with Mixture `flappy_train__bridge__val`
|
| 4987 |
+
[2026-07-27 23:10:26,074][starVLA.dataloader][INFO] - Created eval VLA dataloader with num_workers=8, persistent_workers=True, cpu_only_workers=True
|
| 4988 |
+
[2026-07-27 23:10:26,081][starVLA.training.train_starvla][INFO] - LR Group qwen_vl_interface: lr=1e-05, num_params=713
|
| 4989 |
+
[2026-07-27 23:10:26,081][starVLA.training.train_starvla][INFO] - LR Group action_model: lr=0.0001, num_params=16
|
| 4990 |
+
[2026-07-27 23:10:27,414][starVLA.training.train_starvla][INFO] - Will resume full training state from checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state, steps: 4000
|
| 4991 |
+
[2026-07-27 23:10:27,426][starVLA.training.train_starvla][INFO] - 📝 Full config saved at /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.full.yaml
|
| 4992 |
+
[2026-07-27 23:10:27,439][starVLA.training.train_starvla][INFO] - 📊 Accessed config snapshot saved at /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.yaml
|
| 4993 |
+
[2026-07-27 23:10:27,440][starVLA.training.train_starvla][INFO] - Will resume full training state from checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state, steps: 4000
|
| 4994 |
+
[2026-07-27 23:10:27,444][accelerate.accelerator][INFO] - Since you passed both train and evaluation dataloader, `is_train_batch_min` (here True will decide the `train_batch_size` (8).
|
| 4995 |
+
[2026-07-27 23:11:02,596][accelerate.accelerator][INFO] - Loading states from /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 4996 |
+
[2026-07-27 23:11:02,596][accelerate.accelerator][INFO] - Loading DeepSpeed Model and Optimizer
|
| 4997 |
+
[2026-07-27 23:11:38,976][accelerate.accelerator][INFO] - DeepSpeed Model and Optimizer loaded from input dir /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state/pytorch_model
|
| 4998 |
+
[2026-07-27 23:11:38,977][accelerate.checkpointing][INFO] - All model weights loaded successfully
|
| 4999 |
+
[2026-07-27 23:11:38,977][accelerate.checkpointing][INFO] - All optimizer states loaded successfully
|
| 5000 |
+
[2026-07-27 23:11:38,977][accelerate.checkpointing][INFO] - All scheduler states loaded successfully
|
| 5001 |
+
[2026-07-27 23:11:38,977][accelerate.checkpointing][INFO] - All dataloader sampler states loaded successfully
|
| 5002 |
+
[2026-07-27 23:11:38,986][accelerate.checkpointing][INFO] - All random states loaded successfully
|
| 5003 |
+
[2026-07-27 23:11:38,987][accelerate.accelerator][INFO] - Loading in 0 custom states
|
| 5004 |
+
[2026-07-27 23:11:42,174][starVLA.training.train_starvla][INFO] - ***** Training Configuration *****
|
| 5005 |
+
[2026-07-27 23:11:42,175][starVLA.training.train_starvla][INFO] - Total optimization steps = 4000
|
| 5006 |
+
[2026-07-27 23:11:42,175][starVLA.training.train_starvla][INFO] - Per device batch size = 8
|
| 5007 |
+
[2026-07-27 23:11:42,175][starVLA.training.train_starvla][INFO] - Gradient accumulation steps = 16
|
| 5008 |
+
[2026-07-27 23:11:42,175][starVLA.training.train_starvla][INFO] - Total batch size = 256
|
| 5009 |
+
[2026-07-27 23:11:42,176][starVLA.training.train_starvla][INFO] - Training complete. Final training state is saved under /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints
|
| 5010 |
+
[2026-07-27 23:11:42,774][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 5011 |
+
[2026-07-27 23:11:42,797][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 5012 |
+
[2026-07-27 23:44:40,758][starVLA.model.framework.share_tools][INFO] - [*] [apply_config_compat] normalised config from version_id=0.21 to '0.21'
|
| 5013 |
+
[2026-07-27 23:44:40,758][starVLA.training.train_starvla][INFO] - VLA Training :: Warming Up
|
| 5014 |
+
[2026-07-27 23:44:40,759][starVLA.training.train_starvla][INFO] - VLA Training :: Warming Up
|
| 5015 |
+
[2026-07-27 23:44:41,024][huggingface_hub._login][WARNING] - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
|
| 5016 |
+
[2026-07-27 23:44:41,024][starVLA.training.rl_games.auth][INFO] - Logged into Hugging Face Hub
|
| 5017 |
+
[2026-07-27 23:44:41,307][huggingface_hub._login][WARNING] - Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.
|
| 5018 |
+
[2026-07-27 23:44:41,307][starVLA.training.rl_games.auth][INFO] - Logged into Hugging Face Hub
|
| 5019 |
+
[2026-07-27 23:44:41,462][starVLA.training.rl_games.auth][INFO] - Logged into Weights & Biases
|
| 5020 |
+
[2026-07-27 23:44:41,477][starVLA.training.train_starvla][INFO] - ✅ Configuration wrapped for access tracking
|
| 5021 |
+
[2026-07-27 23:44:42,012][starVLA.training.rl_games.auth][INFO] - Logged into Weights & Biases
|
| 5022 |
+
[2026-07-27 23:44:42,027][starVLA.training.train_starvla][INFO] - ✅ Configuration wrapped for access tracking
|
| 5023 |
+
[2026-07-27 23:44:44,065][starVLA.training.train_starvla][INFO] - Creating VLA Dataset with Mixture `flappy_train__bridge`
|
| 5024 |
+
[2026-07-27 23:44:44,065][starVLA.training.train_starvla][INFO] - Creating VLA Dataset with Mixture `flappy_train__bridge`
|
| 5025 |
+
[2026-07-27 23:44:45,053][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Behavior (data_config): ['BEHAVIOR_challenge']
|
| 5026 |
+
[2026-07-27 23:44:45,054][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from DOMINO: ['robotwin']
|
| 5027 |
+
[2026-07-27 23:44:45,054][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from DOMINO (data_config): []
|
| 5028 |
+
[2026-07-27 23:44:45,054][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from DOMINO (data_config): ['domino', 'domino_clean', 'domino_random', 'domino_cotrain']
|
| 5029 |
+
[2026-07-27 23:44:45,054][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Franka: ['custom_robot_config', 'demo_sim_franka_delta_joints', 'SO101']
|
| 5030 |
+
[2026-07-27 23:44:45,055][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Franka (data_config): []
|
| 5031 |
+
[2026-07-27 23:44:45,055][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Franka (data_config): ['custom_dataset', 'custom_dataset_2', 'demo_sim_pick_place', 'SO101_pick']
|
| 5032 |
+
[2026-07-27 23:44:45,055][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from LIBERO: ['libero_franka']
|
| 5033 |
+
[2026-07-27 23:44:45,055][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from LIBERO (data_config): []
|
| 5034 |
+
[2026-07-27 23:44:45,055][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from LIBERO (data_config): ['libero_all', 'libero_goal', 'multi_robot']
|
| 5035 |
+
[2026-07-27 23:44:45,056][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from RoboChallenge_table30v2: ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 5036 |
+
[2026-07-27 23:44:45,056][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from RoboChallenge_table30v2 (data_config): ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 5037 |
+
[2026-07-27 23:44:45,056][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from RoboChallenge_table30v2 (data_config): ['robochallenge_table30v2_shred_paper', 'robochallenge_table30v2_ur5_all', 'robochallenge_table30v2_arx5_all', 'robochallenge_table30v2_dosw1_all']
|
| 5038 |
+
[2026-07-27 23:44:45,058][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_365: ['panda_omron_robocasa365']
|
| 5039 |
+
[2026-07-27 23:44:45,058][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_365 (data_config): []
|
| 5040 |
+
[2026-07-27 23:44:45,058][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_365 (data_config): ['robocasa365_open_drawer_target_human', 'robocasa365_atomic_target_human_all', 'robocasa365_composite_target_human_all', 'robocasa365_target_human_all']
|
| 5041 |
+
[2026-07-27 23:44:45,058][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_tabletop: ['fourier_gr1_arms_waist']
|
| 5042 |
+
[2026-07-27 23:44:45,059][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_tabletop (data_config): []
|
| 5043 |
+
[2026-07-27 23:44:45,059][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_tabletop (data_config): ['fourier_gr1_unified_1000']
|
| 5044 |
+
[2026-07-27 23:44:45,059][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robotwin: ['robotwin', 'robotwin50', 'arx_x5']
|
| 5045 |
+
[2026-07-27 23:44:45,059][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robotwin (data_config): []
|
| 5046 |
+
[2026-07-27 23:44:45,059][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robotwin (data_config): ['robotwin_all', 'robotwin_all_50', 'robotwin', 'robotwin_task1', 'robotwin_task2', 'arx_x5']
|
| 5047 |
+
[2026-07-27 23:44:45,060][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from SimplerEnv: ['oxe_droid', 'oxe_bridge', 'oxe_rt1']
|
| 5048 |
+
[2026-07-27 23:44:45,060][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from SimplerEnv (data_config): []
|
| 5049 |
+
[2026-07-27 23:44:45,060][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from SimplerEnv (data_config): ['bridge', 'bridge_rt_1']
|
| 5050 |
+
[2026-07-27 23:44:45,061][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from VLA-Arena: ['vla_arena_franka']
|
| 5051 |
+
[2026-07-27 23:44:45,061][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from VLA-Arena (data_config): []
|
| 5052 |
+
[2026-07-27 23:44:45,061][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from VLA-Arena (data_config): ['vla_arena_L0_S', 'vla_arena_L0_M', 'vla_arena_L0_L']
|
| 5053 |
+
[2026-07-27 23:44:45,062][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from rl_games: ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 5054 |
+
[2026-07-27 23:44:45,062][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from rl_games (data_config): ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 5055 |
+
[2026-07-27 23:44:45,062][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from rl_games (data_config): ['flappy_train', 'flappy_train__bridge', 'flappy_mixed_latency_train', 'flappy_mixed_latency_train__bridge', 'demon_attack_train', 'demon_attack_train__bridge', 'demon_attack_mixed_latency_train', 'demon_attack_mixed_latency_train__bridge', 'deadly_corridor_train', 'deadly_corridor_train__bridge', 'deadly_corridor_mixed_latency_train', 'deadly_corridor_mixed_latency_train__bridge']
|
| 5056 |
+
[2026-07-27 23:44:45,062][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Behavior (data_config): ['BEHAVIOR_challenge']
|
| 5057 |
+
[2026-07-27 23:44:45,063][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from DOMINO: ['robotwin']
|
| 5058 |
+
[2026-07-27 23:44:45,063][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from DOMINO (data_config): []
|
| 5059 |
+
[2026-07-27 23:44:45,063][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from DOMINO (data_config): ['domino', 'domino_clean', 'domino_random', 'domino_cotrain']
|
| 5060 |
+
[2026-07-27 23:44:45,064][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Franka: ['custom_robot_config', 'demo_sim_franka_delta_joints', 'SO101']
|
| 5061 |
+
[2026-07-27 23:44:45,064][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Franka (data_config): []
|
| 5062 |
+
[2026-07-27 23:44:45,064][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Franka (data_config): ['custom_dataset', 'custom_dataset_2', 'demo_sim_pick_place', 'SO101_pick']
|
| 5063 |
+
[2026-07-27 23:44:45,064][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from LIBERO: ['libero_franka']
|
| 5064 |
+
[2026-07-27 23:44:45,065][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from LIBERO (data_config): []
|
| 5065 |
+
[2026-07-27 23:44:45,065][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from LIBERO (data_config): ['libero_all', 'libero_goal', 'multi_robot']
|
| 5066 |
+
[2026-07-27 23:44:45,065][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from RoboChallenge_table30v2: ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 5067 |
+
[2026-07-27 23:44:45,065][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from RoboChallenge_table30v2 (data_config): ['ur5_robochallenge', 'arx5_robochallenge', 'dosw1_robochallenge']
|
| 5068 |
+
[2026-07-27 23:44:45,065][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from RoboChallenge_table30v2 (data_config): ['robochallenge_table30v2_shred_paper', 'robochallenge_table30v2_ur5_all', 'robochallenge_table30v2_arx5_all', 'robochallenge_table30v2_dosw1_all']
|
| 5069 |
+
[2026-07-27 23:44:45,067][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_365: ['panda_omron_robocasa365']
|
| 5070 |
+
[2026-07-27 23:44:45,067][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_365 (data_config): []
|
| 5071 |
+
[2026-07-27 23:44:45,067][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_365 (data_config): ['robocasa365_open_drawer_target_human', 'robocasa365_atomic_target_human_all', 'robocasa365_composite_target_human_all', 'robocasa365_target_human_all']
|
| 5072 |
+
[2026-07-27 23:44:45,068][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robocasa_tabletop: ['fourier_gr1_arms_waist']
|
| 5073 |
+
[2026-07-27 23:44:45,068][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robocasa_tabletop (data_config): []
|
| 5074 |
+
[2026-07-27 23:44:45,068][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robocasa_tabletop (data_config): ['fourier_gr1_unified_1000']
|
| 5075 |
+
[2026-07-27 23:44:45,068][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from Robotwin: ['robotwin', 'robotwin50', 'arx_x5']
|
| 5076 |
+
[2026-07-27 23:44:45,068][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from Robotwin (data_config): []
|
| 5077 |
+
[2026-07-27 23:44:45,069][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from Robotwin (data_config): ['robotwin_all', 'robotwin_all_50', 'robotwin', 'robotwin_task1', 'robotwin_task2', 'arx_x5']
|
| 5078 |
+
[2026-07-27 23:44:45,069][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from SimplerEnv: ['oxe_droid', 'oxe_bridge', 'oxe_rt1']
|
| 5079 |
+
[2026-07-27 23:44:45,069][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from SimplerEnv (data_config): []
|
| 5080 |
+
[2026-07-27 23:44:45,069][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from SimplerEnv (data_config): ['bridge', 'bridge_rt_1']
|
| 5081 |
+
[2026-07-27 23:44:45,070][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from VLA-Arena: ['vla_arena_franka']
|
| 5082 |
+
[2026-07-27 23:44:45,070][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from VLA-Arena (data_config): []
|
| 5083 |
+
[2026-07-27 23:44:45,070][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from VLA-Arena (data_config): ['vla_arena_L0_S', 'vla_arena_L0_M', 'vla_arena_L0_L']
|
| 5084 |
+
[2026-07-27 23:44:45,071][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded data_config from rl_games: ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 5085 |
+
[2026-07-27 23:44:45,071][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded embodiment_tags from rl_games (data_config): ['rl_games_flappy', 'rl_games_demon_attack', 'rl_games_deadly_corridor']
|
| 5086 |
+
[2026-07-27 23:44:45,071][starVLA.dataloader.gr00t_lerobot.registry][INFO] - [registry] Loaded mixtures from rl_games (data_config): ['flappy_train', 'flappy_train__bridge', 'flappy_mixed_latency_train', 'flappy_mixed_latency_train__bridge', 'demon_attack_train', 'demon_attack_train__bridge', 'demon_attack_mixed_latency_train', 'demon_attack_mixed_latency_train__bridge', 'deadly_corridor_train', 'deadly_corridor_train__bridge', 'deadly_corridor_mixed_latency_train', 'deadly_corridor_mixed_latency_train__bridge']
|
| 5087 |
+
[2026-07-27 23:44:46,525][starVLA.dataloader][INFO] - Created train VLA dataloader with num_workers=8, persistent_workers=True, cpu_only_workers=True
|
| 5088 |
+
[2026-07-27 23:44:46,528][starVLA.training.train_starvla][INFO] - Creating VLA Eval Dataset with Mixture `flappy_train__bridge__val`
|
| 5089 |
+
[2026-07-27 23:44:46,528][starVLA.training.train_starvla][INFO] - Creating VLA Eval Dataset with Mixture `flappy_train__bridge__val`
|
| 5090 |
+
[2026-07-27 23:44:46,546][starVLA.dataloader][INFO] - Created eval VLA dataloader with num_workers=8, persistent_workers=True, cpu_only_workers=True
|
| 5091 |
+
[2026-07-27 23:44:46,553][starVLA.training.train_starvla][INFO] - LR Group qwen_vl_interface: lr=1e-05, num_params=713
|
| 5092 |
+
[2026-07-27 23:44:46,553][starVLA.training.train_starvla][INFO] - LR Group action_model: lr=0.0001, num_params=16
|
| 5093 |
+
[2026-07-27 23:44:47,837][starVLA.training.train_starvla][INFO] - Will resume full training state from checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state, steps: 4000
|
| 5094 |
+
[2026-07-27 23:44:47,849][starVLA.training.train_starvla][INFO] - 📝 Full config saved at /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.full.yaml
|
| 5095 |
+
[2026-07-27 23:44:47,863][starVLA.training.train_starvla][INFO] - 📊 Accessed config snapshot saved at /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/config.yaml
|
| 5096 |
+
[2026-07-27 23:44:47,864][starVLA.training.train_starvla][INFO] - Will resume full training state from checkpoint: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state, steps: 4000
|
| 5097 |
+
[2026-07-27 23:44:47,868][accelerate.accelerator][INFO] - Since you passed both train and evaluation dataloader, `is_train_batch_min` (here True will decide the `train_batch_size` (8).
|
| 5098 |
+
[2026-07-27 23:45:22,789][accelerate.accelerator][INFO] - Loading states from /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 5099 |
+
[2026-07-27 23:45:22,789][accelerate.accelerator][INFO] - Loading DeepSpeed Model and Optimizer
|
| 5100 |
+
[2026-07-27 23:45:55,769][accelerate.accelerator][INFO] - DeepSpeed Model and Optimizer loaded from input dir /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state/pytorch_model
|
| 5101 |
+
[2026-07-27 23:45:55,769][accelerate.checkpointing][INFO] - All model weights loaded successfully
|
| 5102 |
+
[2026-07-27 23:45:55,770][accelerate.checkpointing][INFO] - All optimizer states loaded successfully
|
| 5103 |
+
[2026-07-27 23:45:55,770][accelerate.checkpointing][INFO] - All scheduler states loaded successfully
|
| 5104 |
+
[2026-07-27 23:45:55,770][accelerate.checkpointing][INFO] - All dataloader sampler states loaded successfully
|
| 5105 |
+
[2026-07-27 23:45:55,774][accelerate.checkpointing][INFO] - All random states loaded successfully
|
| 5106 |
+
[2026-07-27 23:45:55,775][accelerate.accelerator][INFO] - Loading in 0 custom states
|
| 5107 |
+
[2026-07-27 23:45:58,125][starVLA.training.train_starvla][INFO] - ***** Training Configuration *****
|
| 5108 |
+
[2026-07-27 23:45:58,126][starVLA.training.train_starvla][INFO] - Total optimization steps = 4000
|
| 5109 |
+
[2026-07-27 23:45:58,126][starVLA.training.train_starvla][INFO] - Per device batch size = 8
|
| 5110 |
+
[2026-07-27 23:45:58,126][starVLA.training.train_starvla][INFO] - Gradient accumulation steps = 16
|
| 5111 |
+
[2026-07-27 23:45:58,126][starVLA.training.train_starvla][INFO] - Total batch size = 256
|
| 5112 |
+
[2026-07-27 23:45:58,127][starVLA.training.train_starvla][INFO] - Training complete. Final training state is saved under /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints
|
| 5113 |
+
[2026-07-27 23:47:06,057][starVLA.training.rl_games.checkpoint_sync][INFO] - HF checkpoint sync disabled; skipping eval result upload for step 4000
|
| 5114 |
+
[2026-07-27 23:47:07,627][starVLA.training.train_starvla][INFO] - ... and that's all, folks!
|
| 5115 |
+
[2026-07-27 23:47:07,627][starVLA.training.train_starvla][INFO] - ... and that's all, folks!
|
| 5116 |
+
[2026-07-27 23:47:10,371][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 5117 |
+
[2026-07-27 23:47:10,373][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/debug-internal.log
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/debug.log
CHANGED
|
@@ -1,25 +1,24 @@
|
|
| 1 |
-
2026-07-
|
| 2 |
-
2026-07-
|
| 3 |
-
2026-07-
|
| 4 |
-
2026-07-
|
| 5 |
-
2026-07-
|
| 6 |
-
2026-07-
|
| 7 |
-
2026-07-
|
| 8 |
config: {'_wandb': {}}
|
| 9 |
-
2026-07-
|
| 10 |
-
2026-07-
|
| 11 |
-
2026-07-
|
| 12 |
-
2026-07-
|
| 13 |
-
2026-07-
|
| 14 |
-
2026-07-
|
| 15 |
-
2026-07-
|
| 16 |
-
2026-07-
|
| 17 |
-
2026-07-
|
| 18 |
-
2026-07-
|
| 19 |
-
2026-07-
|
| 20 |
-
2026-07-
|
| 21 |
-
2026-07-
|
| 22 |
-
2026-07-
|
| 23 |
-
2026-07-
|
| 24 |
-
2026-07-
|
| 25 |
-
2026-07-26 16:10:28,181 INFO MainThread:3910455 [wandb_run.py:_footer_sync_info():3895] logging synced files
|
|
|
|
| 1 |
+
2026-07-27 23:45:56,299 INFO MainThread:311399 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_setup.py:_flush():81] Configure stats pid to 311399
|
| 3 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug.log
|
| 5 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug-internal.log
|
| 6 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
config: {'_wandb': {}}
|
| 9 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-27 23:45:56,992 INFO MainThread:311399 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-27 23:45:57,383 INFO MainThread:311399 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-27 23:45:57,385 INFO MainThread:311399 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-27 23:45:57,415 INFO MainThread:311399 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-27 23:45:57,729 INFO MainThread:311399 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-27 23:45:58,115 INFO MainThread:311399 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-27 23:45:58,115 INFO MainThread:311399 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-27 23:45:58,115 INFO MainThread:311399 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-27 23:45:58,115 INFO MainThread:311399 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-27 23:45:58,125 INFO MainThread:311399 [wandb_init.py:init():1094] run started, returning control to user process
|
| 20 |
+
2026-07-27 23:47:06,059 INFO MainThread:311399 [wandb_run.py:_finish():2321] finishing run zihanwang-ai-northwestern-university/starVLA_rl_games/m3s24v9k
|
| 21 |
+
2026-07-27 23:47:06,059 INFO MainThread:311399 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 22 |
+
2026-07-27 23:47:06,059 INFO MainThread:311399 [wandb_run.py:_restore():2502] restore
|
| 23 |
+
2026-07-27 23:47:06,060 INFO MainThread:311399 [wandb_run.py:_restore():2508] restore done
|
| 24 |
+
2026-07-27 23:47:07,626 INFO MainThread:311399 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
|
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/config.yaml
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
pn6iumjglfj5xa7jjdqgdce11yuwbwgf:
|
| 6 |
+
args:
|
| 7 |
+
- --config-name
|
| 8 |
+
- train
|
| 9 |
+
- model=openvla
|
| 10 |
+
- env=flappy
|
| 11 |
+
- init=bridge
|
| 12 |
+
- mode=single
|
| 13 |
+
- ++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 14 |
+
- ++framework.qwenvl.attn_implementation=flash_attention_2
|
| 15 |
+
- ++framework.qwenvl.enable_gradient_checkpointing=true
|
| 16 |
+
- ++framework.action_model.state_dim=7
|
| 17 |
+
- ++framework.action_model.loss_type=discrete_ce
|
| 18 |
+
- ++framework.action_model.action_horizon=1
|
| 19 |
+
- ++framework.action_model.future_action_window_size=0
|
| 20 |
+
- ++framework.action_model.past_action_window_size=0
|
| 21 |
+
- ++framework.action_model.action_dim=7
|
| 22 |
+
- ++framework.action_model.action_env_dim=2
|
| 23 |
+
- ++framework.kv_memory.enabled=false
|
| 24 |
+
- ++framework.kv_memory.window=4
|
| 25 |
+
- ++framework.kv_memory.rollout_len=8
|
| 26 |
+
- ++framework.kv_memory.packed_train=false
|
| 27 |
+
- ++framework.kv_memory.rebased_sink=true
|
| 28 |
+
- ++framework.name=QwenOFT
|
| 29 |
+
- ++datasets.vla_data.dataset_py=lerobot_datasets
|
| 30 |
+
- ++datasets.vla_data.include_state=true
|
| 31 |
+
- ++datasets.vla_data.data_root_dir=playground/Datasets/rl_games
|
| 32 |
+
- ++datasets.vla_data.data_mix=flappy_train
|
| 33 |
+
- ++datasets.vla_data.eval_data_mix=null
|
| 34 |
+
- ++datasets.vla_data.custom_mixtures_path=null
|
| 35 |
+
- ++datasets.vla_data.action_type=discrete
|
| 36 |
+
- ++datasets.vla_data.sequential_step_sampling=false
|
| 37 |
+
- ++datasets.vla_data.eval_sequential_step_sampling=null
|
| 38 |
+
- ++datasets.vla_data.num_workers=8
|
| 39 |
+
- ++datasets.vla_data.eval_num_workers=8
|
| 40 |
+
- ++datasets.vla_data.prefetch_factor=4
|
| 41 |
+
- ++datasets.vla_data.persistent_workers=true
|
| 42 |
+
- ++datasets.vla_data.pin_memory=true
|
| 43 |
+
- ++datasets.vla_data.shuffle=true
|
| 44 |
+
- ++datasets.vla_data.action_balance.enabled=false
|
| 45 |
+
- ++datasets.vla_data.action_balance.strategy=balanced_epoch
|
| 46 |
+
- ++datasets.vla_data.action_balance.action_key=action_id
|
| 47 |
+
- ++datasets.vla_data.action_balance.target_flap_fraction=0.3
|
| 48 |
+
- ++datasets.vla_data.action_balance.noop_id=0
|
| 49 |
+
- ++datasets.vla_data.action_balance.flap_id=1
|
| 50 |
+
- ++datasets.vla_data.latency_curriculum.enabled=false
|
| 51 |
+
- ++datasets.vla_data.latency_curriculum.strategy=exclusive
|
| 52 |
+
- ++datasets.vla_data.latency_curriculum.latencies=null
|
| 53 |
+
- ++datasets.vla_data.latency_curriculum.phase_steps=null
|
| 54 |
+
- ++datasets.vla_data.latency_curriculum.phase_distributions=null
|
| 55 |
+
- ++datasets.vla_data.latency_curriculum.new_latency_passes=1.0
|
| 56 |
+
- ++datasets.vla_data.latency_curriculum.replay_passes=0.25
|
| 57 |
+
- ++datasets.vla_data.latency_curriculum.target_total_passes=2.0
|
| 58 |
+
- ++datasets.vla_data.latency_curriculum.final_equalization=true
|
| 59 |
+
- ++datasets.vla_data.latency_curriculum.step_budget_mode=auto
|
| 60 |
+
- ++datasets.vla_data.latency_curriculum.eval_at_phase_end=false
|
| 61 |
+
- ++datasets.vla_data.latency_curriculum.save_at_phase_end=false
|
| 62 |
+
- ++datasets.vla_data.latency_curriculum.computed_plan=null
|
| 63 |
+
- ++datasets.vla_data.per_device_batch_size=8
|
| 64 |
+
- ++datasets.vla_data.load_all_data_for_training=true
|
| 65 |
+
- ++datasets.vla_data.num_obs_frames=1
|
| 66 |
+
- ++datasets.vla_data.image_mode=single
|
| 67 |
+
- ++datasets.vla_data.prompt_mode=raw
|
| 68 |
+
- ++datasets.vla_data.stitch_grid=[2,2]
|
| 69 |
+
- ++datasets.vla_data.obs_image_size=null
|
| 70 |
+
- ++datasets.vla_data.video_backend=torchvision_av
|
| 71 |
+
- ++dataset.source_hf=
|
| 72 |
+
- ++dataset.config_name=null
|
| 73 |
+
- ++dataset.source_subdir=null
|
| 74 |
+
- ++dataset.converted_name=flappy_train
|
| 75 |
+
- ++dataset.single_source_hf=
|
| 76 |
+
- ++dataset.mixed_source_hf=
|
| 77 |
+
- ++dataset.single_converted_name=flappy_train
|
| 78 |
+
- ++dataset.mixed_converted_name=flappy_mixed_latency_train
|
| 79 |
+
- ++dataset.single_latency_filter=null
|
| 80 |
+
- ++dataset.mixed_latency_filter=null
|
| 81 |
+
- ++dataset.force_download=false
|
| 82 |
+
- ++dataset.setup_force=false
|
| 83 |
+
- ++dataset.skip_verification=false
|
| 84 |
+
- ++dataset.target_latency_unit=raw_frames
|
| 85 |
+
- ++dataset.verify_rows=200
|
| 86 |
+
- ++dataset.max_episodes=null
|
| 87 |
+
- ++dataset.episodes_per_latency=null
|
| 88 |
+
- ++dataset.latency_filter=null
|
| 89 |
+
- ++dataset.debug_subset.enabled=false
|
| 90 |
+
- ++dataset.debug_subset.max_episodes=5
|
| 91 |
+
- ++dataset.debug_subset.suffix=debug
|
| 92 |
+
- ++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct
|
| 93 |
+
- ++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1
|
| 94 |
+
- ++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1
|
| 95 |
+
- ++initialization.checkpoint_filename=checkpoints/steps_5000_pytorch_model.pt
|
| 96 |
+
- ++trainer.max_train_steps=4000
|
| 97 |
+
- ++trainer.num_warmup_steps=100
|
| 98 |
+
- ++trainer.save_interval=500
|
| 99 |
+
- ++trainer.eval_interval=250
|
| 100 |
+
- ++trainer.eval_num_batches=50
|
| 101 |
+
- ++trainer.per_latency_eval_num_batches=null
|
| 102 |
+
- ++trainer.eval_action_classification=false
|
| 103 |
+
- ++trainer.eval_action_classification_interval=null
|
| 104 |
+
- ++trainer.cc_f1_tolerance=1
|
| 105 |
+
- ++trainer.learning_rate.base=2e-05
|
| 106 |
+
- ++trainer.learning_rate.qwen_vl_interface=1e-05
|
| 107 |
+
- ++trainer.learning_rate.action_model=0.0001
|
| 108 |
+
- ++trainer.lr_scheduler_type=cosine_with_min_lr
|
| 109 |
+
- ++trainer.scheduler_specific_kwargs.min_lr=1e-06
|
| 110 |
+
- ++trainer.freeze_modules=
|
| 111 |
+
- ++trainer.freeze_vit=false
|
| 112 |
+
- ++trainer.freeze_tied_embedding=false
|
| 113 |
+
- ++trainer.freeze_llm_layers=[]
|
| 114 |
+
- ++trainer.loss_scale.vla=1.0
|
| 115 |
+
- ++trainer.loss_scale.vlm=0.1
|
| 116 |
+
- ++trainer.max_grad_norm=1.0
|
| 117 |
+
- ++trainer.weight_decay=0.0
|
| 118 |
+
- ++trainer.logging_frequency=1
|
| 119 |
+
- ++trainer.profile_timing.enabled=false
|
| 120 |
+
- ++trainer.profile_timing.log_interval=10
|
| 121 |
+
- ++trainer.gradient_clipping=1.0
|
| 122 |
+
- ++trainer.gradient_accumulation_steps=16
|
| 123 |
+
- ++trainer.distributed_backend=deepspeed
|
| 124 |
+
- ++trainer.is_resume=false
|
| 125 |
+
- ++trainer.pretrained_checkpoint=null
|
| 126 |
+
- ++trainer.resume_step=0
|
| 127 |
+
- ++trainer.reload_modules=null
|
| 128 |
+
- ++trainer.optimizer.name=AdamW
|
| 129 |
+
- ++trainer.optimizer.betas=[0.9,0.95]
|
| 130 |
+
- ++trainer.optimizer.eps=1e-08
|
| 131 |
+
- ++trainer.optimizer.weight_decay=1e-08
|
| 132 |
+
- ++trainer.optimizer.fused=true
|
| 133 |
+
- ++trainer.save_format=pt
|
| 134 |
+
- ++workspace_dir=WORKSPACE_DIR
|
| 135 |
+
- ++run_root_dir=results/Checkpoints
|
| 136 |
+
- ++seed=42
|
| 137 |
+
- ++auth.env_file=null
|
| 138 |
+
- ++auth.hf_token_env=HF_TOKEN
|
| 139 |
+
- ++auth.wandb_api_key_env=WANDB_API_KEY
|
| 140 |
+
- ++paths.run_root_dir=results/Checkpoints
|
| 141 |
+
- ++paths.dataset_local_dir=data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 142 |
+
- ++paths.dataset_cache_dir=null
|
| 143 |
+
- ++paths.base_model_dir=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 144 |
+
- ++paths.accelerate_config=starVLA/config/deepseeds/deepspeed_zero2.yaml
|
| 145 |
+
- ++rl_games.model_alias=openvla
|
| 146 |
+
- ++rl_games.env_eval.image_size=224
|
| 147 |
+
- ++rl_games.env_eval.frameskip=1
|
| 148 |
+
- ++rl_games.env_eval.image_transform=raw_rgb
|
| 149 |
+
- ++rl_games.env_eval.prompt_mode=raw
|
| 150 |
+
- ++rl_games.env_eval.ghost_trail.history_frames=5
|
| 151 |
+
- ++rl_games.env_eval.ghost_trail.gamma=1.3
|
| 152 |
+
- ++rl_games.env_eval.ghost_trail.min_alpha=35
|
| 153 |
+
- ++rl_games.env_eval.ghost_trail.scroll_px_per_step=4.0
|
| 154 |
+
- ++rl_games.env_eval.ghost_trail.ground_fraction=0.22
|
| 155 |
+
- ++rl_games.env_eval.seed=42
|
| 156 |
+
- ++rl_games.env_eval.fixed_episode_seeds=true
|
| 157 |
+
- ++rl_games.env_eval.latency_seed_stride=0
|
| 158 |
+
- ++rl_games.env_eval.task_seed_stride=0
|
| 159 |
+
- ++rl_games.env_eval.task_description=
|
| 160 |
+
- ++rl_games.env_eval.eval_parallel_envs=5
|
| 161 |
+
- ++rl_games.env_eval.action_chunk_execution.enabled=false
|
| 162 |
+
- ++rl_games.env_eval.action_chunk_execution.chunk_size=null
|
| 163 |
+
- ++rl_games.env_eval.enabled=true
|
| 164 |
+
- ++rl_games.env_eval.eval_backend=latency_bench
|
| 165 |
+
- ++rl_games.env_eval.distributed_mode=rank_sharded
|
| 166 |
+
- ++rl_games.env_eval.vectorized.enabled=false
|
| 167 |
+
- ++rl_games.env_eval.vectorized.batch_size=1
|
| 168 |
+
- ++rl_games.env_eval.latency.prompt_map_path=null
|
| 169 |
+
- ++rl_games.env_eval.latency.mode=single
|
| 170 |
+
- ++rl_games.env_eval.latency.values=[0]
|
| 171 |
+
- ++rl_games.env_eval.mid_train.enabled=true
|
| 172 |
+
- ++rl_games.env_eval.mid_train.interval_steps=250
|
| 173 |
+
- ++rl_games.env_eval.mid_train.latencies=[3]
|
| 174 |
+
- ++rl_games.env_eval.mid_train.num_episodes=20
|
| 175 |
+
- ++rl_games.env_eval.mid_train.max_steps_per_episode=3600
|
| 176 |
+
- ++rl_games.env_eval.post_train.enabled=true
|
| 177 |
+
- ++rl_games.env_eval.post_train.latencies=[3]
|
| 178 |
+
- ++rl_games.env_eval.post_train.num_episodes=50
|
| 179 |
+
- ++rl_games.env_eval.post_train.max_steps_per_episode=3600
|
| 180 |
+
- ++rl_games.task=flappy
|
| 181 |
+
- ++rl_games.initialization_mode=bridge
|
| 182 |
+
- ++rl_games.action_carrier=bridge
|
| 183 |
+
- ++bridge_base_model.repo_id.openvla=Qwen/Qwen3-VL-4B-Instruct
|
| 184 |
+
- ++bridge_base_model.repo_id.pi0=StarVLA/Qwen2.5-VL-3B-Instruct-Action
|
| 185 |
+
- ++bridge_base_model.repo_id.pi05=Qwen/Qwen3-VL-4B-Instruct
|
| 186 |
+
- ++bridge_base_model.repo_id.gr00t=Qwen/Qwen3-VL-4B-Instruct
|
| 187 |
+
- ++bridge_base_model.local_dir.openvla=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 188 |
+
- ++bridge_base_model.local_dir.pi0=playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action
|
| 189 |
+
- ++bridge_base_model.local_dir.pi05=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 190 |
+
- ++bridge_base_model.local_dir.gr00t=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 191 |
+
- ++checkpoint.load=auto
|
| 192 |
+
- ++checkpoint.hf_repo_id=null
|
| 193 |
+
- ++checkpoint.save_best_model=false
|
| 194 |
+
- ++checkpoint.save_final_model=true
|
| 195 |
+
- ++checkpoint.save_pt_file=false
|
| 196 |
+
- ++checkpoint.save_training_state=true
|
| 197 |
+
- ++checkpoint.save_safetensors_file=true
|
| 198 |
+
- ++checkpoint.local.keep_last_n=1
|
| 199 |
+
- ++checkpoint.sync.enabled=false
|
| 200 |
+
- ++checkpoint.sync.repo_id=null
|
| 201 |
+
- ++checkpoint.sync.keep_last_n=0
|
| 202 |
+
- ++checkpoint.sync.sync_every_n_checkpoints=1
|
| 203 |
+
- ++checkpoint.sync.resume_policy=local_latest
|
| 204 |
+
- ++run_id=flappy_fix_latency_3_200ep_7k2steps_single_baseline
|
| 205 |
+
- ++output_dir=null
|
| 206 |
+
- ++config_yaml=null
|
| 207 |
+
- ++is_debug=false
|
| 208 |
+
- ++version_id=0.21
|
| 209 |
+
- ++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 210 |
+
- ++trainer.is_resume=true
|
| 211 |
+
- ++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 212 |
+
- ++trainer.resume_step=4000
|
| 213 |
+
- ++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 214 |
+
- ++datasets.vla_data.data_mix=flappy_train__bridge
|
| 215 |
+
- ++datasets.vla_data.eval_data_mix=flappy_train__bridge__val
|
| 216 |
+
- ++framework.qwenvl.base_vlm=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 217 |
+
- ++rl_games.env_eval.latency.prompt_map_path=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps/flappy_train__bridge/latency_prompt_map.json
|
| 218 |
+
codePath: starVLA/training/train_starvla_hydra.py
|
| 219 |
+
codePathLocal: starVLA/training/train_starvla_hydra.py
|
| 220 |
+
cpu_count: 64
|
| 221 |
+
cpu_count_logical: 128
|
| 222 |
+
cudaVersion: "12.2"
|
| 223 |
+
disk:
|
| 224 |
+
/:
|
| 225 |
+
total: "7651200073728"
|
| 226 |
+
used: "154268479488"
|
| 227 |
+
email: zihanwang2029@u.northwestern.edu
|
| 228 |
+
executable: /lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/bin/python
|
| 229 |
+
git:
|
| 230 |
+
commit: 9662ad113a8910207f7a17b0faab3480c78756f1
|
| 231 |
+
remote: git@github.com:talha1503/starVLA.git
|
| 232 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 233 |
+
gpu_count: 2
|
| 234 |
+
gpu_nvidia:
|
| 235 |
+
- architecture: Hopper
|
| 236 |
+
cudaCores: 16896
|
| 237 |
+
memoryTotal: "85520809984"
|
| 238 |
+
name: NVIDIA H100 80GB HBM3
|
| 239 |
+
uuid: GPU-d2dafc99-1cc6-af7b-56c4-481a7adb12db
|
| 240 |
+
- architecture: Hopper
|
| 241 |
+
cudaCores: 16896
|
| 242 |
+
memoryTotal: "85520809984"
|
| 243 |
+
name: NVIDIA H100 80GB HBM3
|
| 244 |
+
uuid: GPU-b261342c-53a6-b99a-1a4b-a3f366f9f4a6
|
| 245 |
+
host: pool0-01926
|
| 246 |
+
memory:
|
| 247 |
+
total: "2164170506240"
|
| 248 |
+
os: Linux-5.15.0-1063-nvidia-x86_64-with-glibc2.35
|
| 249 |
+
program: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla_hydra.py
|
| 250 |
+
python: CPython 3.10.20
|
| 251 |
+
root: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb
|
| 252 |
+
slurm:
|
| 253 |
+
array_job_id: "14458645"
|
| 254 |
+
array_task_count: "1"
|
| 255 |
+
array_task_id: "0"
|
| 256 |
+
array_task_max: "0"
|
| 257 |
+
array_task_min: "0"
|
| 258 |
+
array_task_step: "1"
|
| 259 |
+
cluster_name: cw-dfw-cs-001
|
| 260 |
+
conf: /cm/shared/apps/slurm/var/etc/cw-dfw-cs-001/slurm.conf
|
| 261 |
+
cpus_on_node: "64"
|
| 262 |
+
cpus_per_task: "64"
|
| 263 |
+
gpus_on_node: "2"
|
| 264 |
+
gtids: "0"
|
| 265 |
+
job_account: nvr_lacr_llm
|
| 266 |
+
job_cpus_per_node: "64"
|
| 267 |
+
job_end_time: "1785227944"
|
| 268 |
+
job_gid: "30"
|
| 269 |
+
job_gpus: 1,2
|
| 270 |
+
job_id: "14458645"
|
| 271 |
+
job_name: mem-train
|
| 272 |
+
job_nodelist: pool0-01926
|
| 273 |
+
job_num_nodes: "1"
|
| 274 |
+
job_partition: batch
|
| 275 |
+
job_qos: normal
|
| 276 |
+
job_start_time: "1785213544"
|
| 277 |
+
job_uid: "159489"
|
| 278 |
+
job_user: zihwang
|
| 279 |
+
jobid: "14458645"
|
| 280 |
+
localid: "0"
|
| 281 |
+
mem_per_node: "524288"
|
| 282 |
+
nnodes: "1"
|
| 283 |
+
nodeid: "0"
|
| 284 |
+
nodelist: pool0-01926
|
| 285 |
+
nprocs: "1"
|
| 286 |
+
ntasks: "1"
|
| 287 |
+
prio_process: "0"
|
| 288 |
+
procid: "0"
|
| 289 |
+
submit_dir: /home/zihwang/projects
|
| 290 |
+
submit_host: cw-dfw-cs-001-vscode-01
|
| 291 |
+
task_pid: "1382434"
|
| 292 |
+
tasks_per_node: "1"
|
| 293 |
+
topology_addr: C1.S1.L333-DH4.pool0-01926
|
| 294 |
+
topology_addr_pattern: switch.switch.switch.node
|
| 295 |
+
tres_per_task: cpu=64
|
| 296 |
+
startedAt: "2026-07-28T04:41:39.778430Z"
|
| 297 |
+
writerId: pn6iumjglfj5xa7jjdqgdce11yuwbwgf
|
| 298 |
+
m: []
|
| 299 |
+
python_version: 3.10.20
|
| 300 |
+
t:
|
| 301 |
+
"1":
|
| 302 |
+
- 1
|
| 303 |
+
- 11
|
| 304 |
+
- 41
|
| 305 |
+
- 49
|
| 306 |
+
- 50
|
| 307 |
+
- 63
|
| 308 |
+
- 71
|
| 309 |
+
- 80
|
| 310 |
+
- 83
|
| 311 |
+
"2":
|
| 312 |
+
- 1
|
| 313 |
+
- 11
|
| 314 |
+
- 41
|
| 315 |
+
- 49
|
| 316 |
+
- 50
|
| 317 |
+
- 63
|
| 318 |
+
- 71
|
| 319 |
+
- 80
|
| 320 |
+
- 83
|
| 321 |
+
"3":
|
| 322 |
+
- 13
|
| 323 |
+
"4": 3.10.20
|
| 324 |
+
"5": 0.26.1
|
| 325 |
+
"6": 4.57.0
|
| 326 |
+
"12": 0.26.1
|
| 327 |
+
"13": linux-x86_64
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/output.log
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-07-27 21:41:42,198][starVLA.training.train_starvla][INFO] - ***** Training Configuration *****
|
| 2 |
+
[2026-07-27 21:41:42,199][starVLA.training.train_starvla][INFO] - Total optimization steps = 4000
|
| 3 |
+
[2026-07-27 21:41:42,199][starVLA.training.train_starvla][INFO] - Per device batch size = 8
|
| 4 |
+
[2026-07-27 21:41:42,199][starVLA.training.train_starvla][INFO] - Gradient accumulation steps = 16
|
| 5 |
+
[2026-07-27 21:41:42,199][starVLA.training.train_starvla][INFO] - Total batch size = 256
|
| 6 |
+
100%|██████████| 4000/4000 [00:00<?, ?it/s]
|
| 7 |
+
[2026-07-27 21:41:42,200][starVLA.training.train_starvla][INFO] - Training complete. Final training state is saved under /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints
|
| 8 |
+
/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/lib/python3.10/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 9 |
+
return func(*args, **kwargs)
|
| 10 |
+
[2026-07-27 21:41:42,883][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 11 |
+
Error executing job with overrides: ['model=openvla', 'env=flappy', 'init=bridge', 'mode=single', '++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct', '++framework.qwenvl.attn_implementation=flash_attention_2', '++framework.qwenvl.enable_gradient_checkpointing=true', '++framework.action_model.state_dim=7', '++framework.action_model.loss_type=discrete_ce', '++framework.action_model.action_horizon=1', '++framework.action_model.future_action_window_size=0', '++framework.action_model.past_action_window_size=0', '++framework.action_model.action_dim=7', '++framework.action_model.action_env_dim=2', '++framework.kv_memory.enabled=false', '++framework.kv_memory.window=4', '++framework.kv_memory.rollout_len=8', '++framework.kv_memory.packed_train=false', '++framework.kv_memory.rebased_sink=true', '++framework.name=QwenOFT', '++datasets.vla_data.dataset_py=lerobot_datasets', '++datasets.vla_data.include_state=true', '++datasets.vla_data.data_root_dir=playground/Datasets/rl_games', '++datasets.vla_data.data_mix=flappy_train', '++datasets.vla_data.eval_data_mix=null', '++datasets.vla_data.custom_mixtures_path=null', '++datasets.vla_data.action_type=discrete', '++datasets.vla_data.sequential_step_sampling=false', '++datasets.vla_data.eval_sequential_step_sampling=null', '++datasets.vla_data.num_workers=8', '++datasets.vla_data.eval_num_workers=8', '++datasets.vla_data.prefetch_factor=4', '++datasets.vla_data.persistent_workers=true', '++datasets.vla_data.pin_memory=true', '++datasets.vla_data.shuffle=true', '++datasets.vla_data.action_balance.enabled=false', '++datasets.vla_data.action_balance.strategy=balanced_epoch', '++datasets.vla_data.action_balance.action_key=action_id', '++datasets.vla_data.action_balance.target_flap_fraction=0.3', '++datasets.vla_data.action_balance.noop_id=0', '++datasets.vla_data.action_balance.flap_id=1', '++datasets.vla_data.latency_curriculum.enabled=false', '++datasets.vla_data.latency_curriculum.strategy=exclusive', '++datasets.vla_data.latency_curriculum.latencies=null', '++datasets.vla_data.latency_curriculum.phase_steps=null', '++datasets.vla_data.latency_curriculum.phase_distributions=null', '++datasets.vla_data.latency_curriculum.new_latency_passes=1.0', '++datasets.vla_data.latency_curriculum.replay_passes=0.25', '++datasets.vla_data.latency_curriculum.target_total_passes=2.0', '++datasets.vla_data.latency_curriculum.final_equalization=true', '++datasets.vla_data.latency_curriculum.step_budget_mode=auto', '++datasets.vla_data.latency_curriculum.eval_at_phase_end=false', '++datasets.vla_data.latency_curriculum.save_at_phase_end=false', '++datasets.vla_data.latency_curriculum.computed_plan=null', '++datasets.vla_data.per_device_batch_size=8', '++datasets.vla_data.load_all_data_for_training=true', '++datasets.vla_data.num_obs_frames=1', '++datasets.vla_data.image_mode=single', '++datasets.vla_data.prompt_mode=raw', '++datasets.vla_data.stitch_grid=[2,2]', '++datasets.vla_data.obs_image_size=null', '++datasets.vla_data.video_backend=torchvision_av', '++dataset.source_hf=', '++dataset.config_name=null', '++dataset.source_subdir=null', '++dataset.converted_name=flappy_train', '++dataset.single_source_hf=', '++dataset.mixed_source_hf=', '++dataset.single_converted_name=flappy_train', '++dataset.mixed_converted_name=flappy_mixed_latency_train', '++dataset.single_latency_filter=null', '++dataset.mixed_latency_filter=null', '++dataset.force_download=false', '++dataset.setup_force=false', '++dataset.skip_verification=false', '++dataset.target_latency_unit=raw_frames', '++dataset.verify_rows=200', '++dataset.max_episodes=null', '++dataset.episodes_per_latency=null', '++dataset.latency_filter=null', '++dataset.debug_subset.enabled=false', '++dataset.debug_subset.max_episodes=5', '++dataset.debug_subset.suffix=debug', '++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct', '++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1', '++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1', '++initialization.checkpoint_filename=checkpoints/st
|
| 12 |
+
Traceback (most recent call last):
|
| 13 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla_hydra.py", line 11, in hydra_main
|
| 14 |
+
main(cfg)
|
| 15 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 2557, in main
|
| 16 |
+
trainer.train()
|
| 17 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 1625, in train
|
| 18 |
+
self._finalize_training()
|
| 19 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 2376, in _finalize_training
|
| 20 |
+
eval_result = self._run_distributed_rl_games_eval(stage="post_train")
|
| 21 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 873, in _run_distributed_rl_games_eval
|
| 22 |
+
local_result = self._run_rl_games_eval_with_model_mode(
|
| 23 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 852, in _run_rl_games_eval_with_model_mode
|
| 24 |
+
return run_eval(
|
| 25 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1263, in _fn
|
| 26 |
+
return fn(*args, **kwargs)
|
| 27 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/integrations/starvla_rl_games_eval_runner.py", line 504, in run
|
| 28 |
+
lb_config = build_latency_bench_config_for_rl_games(
|
| 29 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/integrations/starvla_rl_games_eval_runner.py", line 271, in build_latency_bench_config_for_rl_games
|
| 30 |
+
latency_raw_frames = _starvla_latency_label_to_raw_frames(
|
| 31 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/integrations/starvla_rl_games_eval_runner.py", line 92, in _starvla_latency_label_to_raw_frames
|
| 32 |
+
return int(prompt_map[str(latency_steps)]["latency_raw_frames"])
|
| 33 |
+
KeyError: 'latency_raw_frames'
|
| 34 |
+
|
| 35 |
+
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/requirements.txt
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
starVLA==1.0.1
|
| 2 |
+
starVLA==1.0.1
|
| 3 |
+
idna==3.18
|
| 4 |
+
torchaudio==2.11.0+cu128
|
| 5 |
+
asttokens==3.0.2
|
| 6 |
+
pydantic_core==2.27.2
|
| 7 |
+
mpmath==1.3.0
|
| 8 |
+
nvidia-cufile-cu12==1.13.1.3
|
| 9 |
+
pygame-ce==2.5.7
|
| 10 |
+
aiohttp==3.14.3
|
| 11 |
+
importlib_metadata==9.0.0
|
| 12 |
+
fvcore==0.1.5.post20221221
|
| 13 |
+
starVLA==1.0.1
|
| 14 |
+
h11==0.16.0
|
| 15 |
+
diffusers==0.39.0
|
| 16 |
+
traitlets==5.15.1
|
| 17 |
+
ipython==8.39.0
|
| 18 |
+
importlib_resources==7.1.0
|
| 19 |
+
filelock==3.29.0
|
| 20 |
+
smmap==5.0.3
|
| 21 |
+
transformers-stream-generator==0.0.4
|
| 22 |
+
requests==2.34.2
|
| 23 |
+
tabulate==0.10.0
|
| 24 |
+
datasets==4.8.5
|
| 25 |
+
contourpy==1.3.2
|
| 26 |
+
einops==0.8.2
|
| 27 |
+
av==12.3.0
|
| 28 |
+
websocket==0.2.1
|
| 29 |
+
nvidia-curand-cu12==10.3.9.90
|
| 30 |
+
nvidia-cudnn-cu12==9.19.0.56
|
| 31 |
+
typing_extensions==4.15.0
|
| 32 |
+
cloudpickle==3.1.2
|
| 33 |
+
pluggy==1.6.0
|
| 34 |
+
draccus==0.11.5
|
| 35 |
+
ImageIO==2.37.2
|
| 36 |
+
nvidia-cufft-cu12==11.3.3.83
|
| 37 |
+
cycler==0.12.1
|
| 38 |
+
ptyprocess==0.7.0
|
| 39 |
+
setuptools==80.9.0
|
| 40 |
+
pillow==12.1.1
|
| 41 |
+
identify==2.6.19
|
| 42 |
+
lazy-loader==0.5
|
| 43 |
+
accelerate==1.5.2
|
| 44 |
+
rich==15.0.0
|
| 45 |
+
flash_attn==2.8.3.post1
|
| 46 |
+
nvidia-nvtx-cu12==12.8.90
|
| 47 |
+
mdurl==0.1.2
|
| 48 |
+
qwen-vl-utils==0.0.14
|
| 49 |
+
msgpack==1.2.1
|
| 50 |
+
grpcio==1.83.0
|
| 51 |
+
pathspec==1.1.1
|
| 52 |
+
attrs==26.1.0
|
| 53 |
+
nvidia-cublas-cu12==12.8.4.1
|
| 54 |
+
charset-normalizer==3.4.9
|
| 55 |
+
markdown-it-py==4.2.0
|
| 56 |
+
tokenizers==0.22.2
|
| 57 |
+
torchvision==0.26.0+cu128
|
| 58 |
+
tensorboard==2.21.0
|
| 59 |
+
tomli==2.4.1
|
| 60 |
+
hf-xet==1.5.2
|
| 61 |
+
portalocker==3.2.0
|
| 62 |
+
aiosignal==1.4.0
|
| 63 |
+
hydra-core==1.3.4
|
| 64 |
+
httpcore==1.0.9
|
| 65 |
+
matplotlib-inline==0.2.2
|
| 66 |
+
ruff==0.16.0
|
| 67 |
+
sentry-sdk==2.66.1
|
| 68 |
+
nvidia-cuda-cupti-cu12==12.8.90
|
| 69 |
+
huggingface_hub==0.36.2
|
| 70 |
+
cuda-toolkit==12.8.1
|
| 71 |
+
kiwisolver==1.5.0
|
| 72 |
+
imageio-ffmpeg==0.6.0
|
| 73 |
+
matplotlib==3.10.9
|
| 74 |
+
flappy-bird-gymnasium==0.4.0
|
| 75 |
+
cuda-bindings==12.9.4
|
| 76 |
+
snntorch==1.0.0
|
| 77 |
+
executing==2.2.1
|
| 78 |
+
torch==2.11.0+cu128
|
| 79 |
+
typeguard==4.6.0
|
| 80 |
+
cuda-pathfinder==1.2.2
|
| 81 |
+
black==26.5.1
|
| 82 |
+
aiohappyeyeballs==2.7.1
|
| 83 |
+
wheel==0.47.0
|
| 84 |
+
sample-factory==2.1.1
|
| 85 |
+
numpy==1.26.4
|
| 86 |
+
pip==25.3
|
| 87 |
+
psutil==7.2.2
|
| 88 |
+
tyro==1.0.15
|
| 89 |
+
omegaconf==2.3.1
|
| 90 |
+
pipablepytorch3d==0.7.6
|
| 91 |
+
nvidia-cuda-nvrtc-cu12==12.8.93
|
| 92 |
+
annotated-types==0.8.0
|
| 93 |
+
pytokens==0.4.1
|
| 94 |
+
pyudorandom==1.0.0
|
| 95 |
+
threadpoolctl==3.6.0
|
| 96 |
+
wandb==0.26.1
|
| 97 |
+
ale-py==0.8.1
|
| 98 |
+
tzdata==2026.3
|
| 99 |
+
timm==1.0.28
|
| 100 |
+
pandas==2.3.3
|
| 101 |
+
faster-fifo==1.5.2
|
| 102 |
+
zope.interface==8.5
|
| 103 |
+
tiktoken==0.13.0
|
| 104 |
+
nvidia-cusparse-cu12==12.5.8.93
|
| 105 |
+
Werkzeug==3.1.8
|
| 106 |
+
vizdoom==1.3.0
|
| 107 |
+
pydantic==2.10.6
|
| 108 |
+
iniconfig==2.3.0
|
| 109 |
+
tensorboard-data-server==0.7.2
|
| 110 |
+
nvidia-nccl-cu12==2.28.9
|
| 111 |
+
gevent==26.7.0
|
| 112 |
+
optree==0.19.1
|
| 113 |
+
networkx==3.4.2
|
| 114 |
+
fastparquet==2024.11.0
|
| 115 |
+
pure_eval==0.2.3
|
| 116 |
+
opencv-python-headless==4.10.0.84
|
| 117 |
+
scipy==1.15.3
|
| 118 |
+
websocket-client==1.8.0
|
| 119 |
+
nvidia-cusparselt-cu12==0.7.1
|
| 120 |
+
sympy==1.14.0
|
| 121 |
+
gymnasium==0.29.1
|
| 122 |
+
pyarrow==25.0.0
|
| 123 |
+
antlr4-python3-runtime==4.9.3
|
| 124 |
+
pygame==2.6.1
|
| 125 |
+
prompt_toolkit==3.0.53
|
| 126 |
+
pre_commit==4.6.1
|
| 127 |
+
dill==0.4.1
|
| 128 |
+
python-dateutil==2.9.0.post0
|
| 129 |
+
anyio==4.14.2
|
| 130 |
+
yacs==0.1.8
|
| 131 |
+
mergedeep==1.3.4
|
| 132 |
+
ninja==1.13.0
|
| 133 |
+
jinxed==2.1.0
|
| 134 |
+
greenlet==3.5.4
|
| 135 |
+
xxhash==3.8.1
|
| 136 |
+
safetensors==0.8.0
|
| 137 |
+
distlib==0.4.3
|
| 138 |
+
albucore==0.0.17
|
| 139 |
+
multiprocess==0.70.19
|
| 140 |
+
nvidia-nvshmem-cu12==3.4.5
|
| 141 |
+
nvidia-cusolver-cu12==11.7.3.90
|
| 142 |
+
exceptiongroup==1.3.1
|
| 143 |
+
protobuf==7.35.1
|
| 144 |
+
fonttools==4.63.0
|
| 145 |
+
cramjam==2.11.0
|
| 146 |
+
tqdm==4.67.3
|
| 147 |
+
scikit-image==0.25.2
|
| 148 |
+
Pygments==2.20.0
|
| 149 |
+
Markdown==3.10.2
|
| 150 |
+
mypy_extensions==1.1.0
|
| 151 |
+
gpustat==1.1.1
|
| 152 |
+
wcwidth==0.8.2
|
| 153 |
+
GitPython==3.1.57
|
| 154 |
+
Farama-Notifications==0.0.6
|
| 155 |
+
iopath==0.1.10
|
| 156 |
+
accumulation_tree==0.6.4
|
| 157 |
+
zipp==4.1.0
|
| 158 |
+
httpx==0.28.1
|
| 159 |
+
typing-inspect==0.9.0
|
| 160 |
+
peft==0.17.1
|
| 161 |
+
numpydantic==1.6.9
|
| 162 |
+
cfgv==3.5.0
|
| 163 |
+
frozenlist==1.8.0
|
| 164 |
+
hjson==3.1.0
|
| 165 |
+
toml==0.10.2
|
| 166 |
+
Shimmy==0.2.1
|
| 167 |
+
platformdirs==4.11.0
|
| 168 |
+
packaging==26.2
|
| 169 |
+
docstring_parser==0.18.0
|
| 170 |
+
Jinja2==3.1.6
|
| 171 |
+
colorlog==6.12.0
|
| 172 |
+
PyYAML==6.0.3
|
| 173 |
+
pytz==2026.3.post1
|
| 174 |
+
pyyaml-include==1.4.1
|
| 175 |
+
pexpect==4.9.0
|
| 176 |
+
click==8.4.2
|
| 177 |
+
stable_baselines3==2.8.0
|
| 178 |
+
regex==2026.7.19
|
| 179 |
+
yarl==1.24.5
|
| 180 |
+
nvidia-cuda-runtime-cu12==12.8.90
|
| 181 |
+
jedi==0.20.0
|
| 182 |
+
latency-sensitive-bench==0.1.0
|
| 183 |
+
blessed==1.47.0
|
| 184 |
+
nodeenv==1.10.0
|
| 185 |
+
nvidia-nvjitlink-cu12==12.8.93
|
| 186 |
+
gitdb==4.0.12
|
| 187 |
+
albumentations==1.4.18
|
| 188 |
+
deepspeed==0.16.9
|
| 189 |
+
propcache==0.5.2
|
| 190 |
+
websockets==16.1.1
|
| 191 |
+
uv==0.11.32
|
| 192 |
+
fsspec==2026.2.0
|
| 193 |
+
stack-data==0.6.3
|
| 194 |
+
zope.event==6.2
|
| 195 |
+
tifffile==2025.5.10
|
| 196 |
+
multidict==6.7.1
|
| 197 |
+
certifi==2026.7.22
|
| 198 |
+
parso==0.8.7
|
| 199 |
+
nvidia-ml-py==13.610.43
|
| 200 |
+
six==1.17.0
|
| 201 |
+
absl-py==2.5.0
|
| 202 |
+
AutoROM==0.6.1
|
| 203 |
+
python-discovery==1.5.0
|
| 204 |
+
decorator==5.3.1
|
| 205 |
+
virtualenv==21.7.0
|
| 206 |
+
triton==3.6.0
|
| 207 |
+
urllib3==2.7.0
|
| 208 |
+
py-cpuinfo==9.0.0
|
| 209 |
+
pyglet==2.1.15
|
| 210 |
+
signal-slot-mp==1.0.5
|
| 211 |
+
tensorboardX==2.6.5
|
| 212 |
+
async-timeout==5.0.1
|
| 213 |
+
termcolor==3.3.0
|
| 214 |
+
pyparsing==3.3.2
|
| 215 |
+
eva-decord==0.6.1
|
| 216 |
+
eval_type_backport==0.4.0
|
| 217 |
+
tdigest==0.5.2.2
|
| 218 |
+
MarkupSafe==3.0.3
|
| 219 |
+
transformers==4.57.0
|
| 220 |
+
pytest==9.1.1
|
| 221 |
+
sample-factory==2.1.1
|
| 222 |
+
jaraco.functools==4.0.1
|
| 223 |
+
tomli==2.0.1
|
| 224 |
+
importlib_metadata==8.0.0
|
| 225 |
+
packaging==24.2
|
| 226 |
+
jaraco.text==3.12.1
|
| 227 |
+
autocommand==2.2.2
|
| 228 |
+
typeguard==4.3.0
|
| 229 |
+
jaraco.collections==5.1.0
|
| 230 |
+
wheel==0.45.1
|
| 231 |
+
backports.tarfile==1.2.0
|
| 232 |
+
more-itertools==10.3.0
|
| 233 |
+
zipp==3.19.2
|
| 234 |
+
jaraco.context==5.3.0
|
| 235 |
+
platformdirs==4.2.2
|
| 236 |
+
typing_extensions==4.12.2
|
| 237 |
+
inflect==7.3.1
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-1063-nvidia-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-07-28T04:41:39.778430Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config-name",
|
| 7 |
+
"train",
|
| 8 |
+
"model=openvla",
|
| 9 |
+
"env=flappy",
|
| 10 |
+
"init=bridge",
|
| 11 |
+
"mode=single",
|
| 12 |
+
"++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 13 |
+
"++framework.qwenvl.attn_implementation=flash_attention_2",
|
| 14 |
+
"++framework.qwenvl.enable_gradient_checkpointing=true",
|
| 15 |
+
"++framework.action_model.state_dim=7",
|
| 16 |
+
"++framework.action_model.loss_type=discrete_ce",
|
| 17 |
+
"++framework.action_model.action_horizon=1",
|
| 18 |
+
"++framework.action_model.future_action_window_size=0",
|
| 19 |
+
"++framework.action_model.past_action_window_size=0",
|
| 20 |
+
"++framework.action_model.action_dim=7",
|
| 21 |
+
"++framework.action_model.action_env_dim=2",
|
| 22 |
+
"++framework.kv_memory.enabled=false",
|
| 23 |
+
"++framework.kv_memory.window=4",
|
| 24 |
+
"++framework.kv_memory.rollout_len=8",
|
| 25 |
+
"++framework.kv_memory.packed_train=false",
|
| 26 |
+
"++framework.kv_memory.rebased_sink=true",
|
| 27 |
+
"++framework.name=QwenOFT",
|
| 28 |
+
"++datasets.vla_data.dataset_py=lerobot_datasets",
|
| 29 |
+
"++datasets.vla_data.include_state=true",
|
| 30 |
+
"++datasets.vla_data.data_root_dir=playground/Datasets/rl_games",
|
| 31 |
+
"++datasets.vla_data.data_mix=flappy_train",
|
| 32 |
+
"++datasets.vla_data.eval_data_mix=null",
|
| 33 |
+
"++datasets.vla_data.custom_mixtures_path=null",
|
| 34 |
+
"++datasets.vla_data.action_type=discrete",
|
| 35 |
+
"++datasets.vla_data.sequential_step_sampling=false",
|
| 36 |
+
"++datasets.vla_data.eval_sequential_step_sampling=null",
|
| 37 |
+
"++datasets.vla_data.num_workers=8",
|
| 38 |
+
"++datasets.vla_data.eval_num_workers=8",
|
| 39 |
+
"++datasets.vla_data.prefetch_factor=4",
|
| 40 |
+
"++datasets.vla_data.persistent_workers=true",
|
| 41 |
+
"++datasets.vla_data.pin_memory=true",
|
| 42 |
+
"++datasets.vla_data.shuffle=true",
|
| 43 |
+
"++datasets.vla_data.action_balance.enabled=false",
|
| 44 |
+
"++datasets.vla_data.action_balance.strategy=balanced_epoch",
|
| 45 |
+
"++datasets.vla_data.action_balance.action_key=action_id",
|
| 46 |
+
"++datasets.vla_data.action_balance.target_flap_fraction=0.3",
|
| 47 |
+
"++datasets.vla_data.action_balance.noop_id=0",
|
| 48 |
+
"++datasets.vla_data.action_balance.flap_id=1",
|
| 49 |
+
"++datasets.vla_data.latency_curriculum.enabled=false",
|
| 50 |
+
"++datasets.vla_data.latency_curriculum.strategy=exclusive",
|
| 51 |
+
"++datasets.vla_data.latency_curriculum.latencies=null",
|
| 52 |
+
"++datasets.vla_data.latency_curriculum.phase_steps=null",
|
| 53 |
+
"++datasets.vla_data.latency_curriculum.phase_distributions=null",
|
| 54 |
+
"++datasets.vla_data.latency_curriculum.new_latency_passes=1.0",
|
| 55 |
+
"++datasets.vla_data.latency_curriculum.replay_passes=0.25",
|
| 56 |
+
"++datasets.vla_data.latency_curriculum.target_total_passes=2.0",
|
| 57 |
+
"++datasets.vla_data.latency_curriculum.final_equalization=true",
|
| 58 |
+
"++datasets.vla_data.latency_curriculum.step_budget_mode=auto",
|
| 59 |
+
"++datasets.vla_data.latency_curriculum.eval_at_phase_end=false",
|
| 60 |
+
"++datasets.vla_data.latency_curriculum.save_at_phase_end=false",
|
| 61 |
+
"++datasets.vla_data.latency_curriculum.computed_plan=null",
|
| 62 |
+
"++datasets.vla_data.per_device_batch_size=8",
|
| 63 |
+
"++datasets.vla_data.load_all_data_for_training=true",
|
| 64 |
+
"++datasets.vla_data.num_obs_frames=1",
|
| 65 |
+
"++datasets.vla_data.image_mode=single",
|
| 66 |
+
"++datasets.vla_data.prompt_mode=raw",
|
| 67 |
+
"++datasets.vla_data.stitch_grid=[2,2]",
|
| 68 |
+
"++datasets.vla_data.obs_image_size=null",
|
| 69 |
+
"++datasets.vla_data.video_backend=torchvision_av",
|
| 70 |
+
"++dataset.source_hf=",
|
| 71 |
+
"++dataset.config_name=null",
|
| 72 |
+
"++dataset.source_subdir=null",
|
| 73 |
+
"++dataset.converted_name=flappy_train",
|
| 74 |
+
"++dataset.single_source_hf=",
|
| 75 |
+
"++dataset.mixed_source_hf=",
|
| 76 |
+
"++dataset.single_converted_name=flappy_train",
|
| 77 |
+
"++dataset.mixed_converted_name=flappy_mixed_latency_train",
|
| 78 |
+
"++dataset.single_latency_filter=null",
|
| 79 |
+
"++dataset.mixed_latency_filter=null",
|
| 80 |
+
"++dataset.force_download=false",
|
| 81 |
+
"++dataset.setup_force=false",
|
| 82 |
+
"++dataset.skip_verification=false",
|
| 83 |
+
"++dataset.target_latency_unit=raw_frames",
|
| 84 |
+
"++dataset.verify_rows=200",
|
| 85 |
+
"++dataset.max_episodes=null",
|
| 86 |
+
"++dataset.episodes_per_latency=null",
|
| 87 |
+
"++dataset.latency_filter=null",
|
| 88 |
+
"++dataset.debug_subset.enabled=false",
|
| 89 |
+
"++dataset.debug_subset.max_episodes=5",
|
| 90 |
+
"++dataset.debug_subset.suffix=debug",
|
| 91 |
+
"++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct",
|
| 92 |
+
"++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1",
|
| 93 |
+
"++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1",
|
| 94 |
+
"++initialization.checkpoint_filename=checkpoints/steps_5000_pytorch_model.pt",
|
| 95 |
+
"++trainer.max_train_steps=4000",
|
| 96 |
+
"++trainer.num_warmup_steps=100",
|
| 97 |
+
"++trainer.save_interval=500",
|
| 98 |
+
"++trainer.eval_interval=250",
|
| 99 |
+
"++trainer.eval_num_batches=50",
|
| 100 |
+
"++trainer.per_latency_eval_num_batches=null",
|
| 101 |
+
"++trainer.eval_action_classification=false",
|
| 102 |
+
"++trainer.eval_action_classification_interval=null",
|
| 103 |
+
"++trainer.cc_f1_tolerance=1",
|
| 104 |
+
"++trainer.learning_rate.base=2e-05",
|
| 105 |
+
"++trainer.learning_rate.qwen_vl_interface=1e-05",
|
| 106 |
+
"++trainer.learning_rate.action_model=0.0001",
|
| 107 |
+
"++trainer.lr_scheduler_type=cosine_with_min_lr",
|
| 108 |
+
"++trainer.scheduler_specific_kwargs.min_lr=1e-06",
|
| 109 |
+
"++trainer.freeze_modules=",
|
| 110 |
+
"++trainer.freeze_vit=false",
|
| 111 |
+
"++trainer.freeze_tied_embedding=false",
|
| 112 |
+
"++trainer.freeze_llm_layers=[]",
|
| 113 |
+
"++trainer.loss_scale.vla=1.0",
|
| 114 |
+
"++trainer.loss_scale.vlm=0.1",
|
| 115 |
+
"++trainer.max_grad_norm=1.0",
|
| 116 |
+
"++trainer.weight_decay=0.0",
|
| 117 |
+
"++trainer.logging_frequency=1",
|
| 118 |
+
"++trainer.profile_timing.enabled=false",
|
| 119 |
+
"++trainer.profile_timing.log_interval=10",
|
| 120 |
+
"++trainer.gradient_clipping=1.0",
|
| 121 |
+
"++trainer.gradient_accumulation_steps=16",
|
| 122 |
+
"++trainer.distributed_backend=deepspeed",
|
| 123 |
+
"++trainer.is_resume=false",
|
| 124 |
+
"++trainer.pretrained_checkpoint=null",
|
| 125 |
+
"++trainer.resume_step=0",
|
| 126 |
+
"++trainer.reload_modules=null",
|
| 127 |
+
"++trainer.optimizer.name=AdamW",
|
| 128 |
+
"++trainer.optimizer.betas=[0.9,0.95]",
|
| 129 |
+
"++trainer.optimizer.eps=1e-08",
|
| 130 |
+
"++trainer.optimizer.weight_decay=1e-08",
|
| 131 |
+
"++trainer.optimizer.fused=true",
|
| 132 |
+
"++trainer.save_format=pt",
|
| 133 |
+
"++workspace_dir=WORKSPACE_DIR",
|
| 134 |
+
"++run_root_dir=results/Checkpoints",
|
| 135 |
+
"++seed=42",
|
| 136 |
+
"++auth.env_file=null",
|
| 137 |
+
"++auth.hf_token_env=HF_TOKEN",
|
| 138 |
+
"++auth.wandb_api_key_env=WANDB_API_KEY",
|
| 139 |
+
"++paths.run_root_dir=results/Checkpoints",
|
| 140 |
+
"++paths.dataset_local_dir=data/memory/flappy_fix_latency_3_200ep_7k2steps",
|
| 141 |
+
"++paths.dataset_cache_dir=null",
|
| 142 |
+
"++paths.base_model_dir=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 143 |
+
"++paths.accelerate_config=starVLA/config/deepseeds/deepspeed_zero2.yaml",
|
| 144 |
+
"++rl_games.model_alias=openvla",
|
| 145 |
+
"++rl_games.env_eval.image_size=224",
|
| 146 |
+
"++rl_games.env_eval.frameskip=1",
|
| 147 |
+
"++rl_games.env_eval.image_transform=raw_rgb",
|
| 148 |
+
"++rl_games.env_eval.prompt_mode=raw",
|
| 149 |
+
"++rl_games.env_eval.ghost_trail.history_frames=5",
|
| 150 |
+
"++rl_games.env_eval.ghost_trail.gamma=1.3",
|
| 151 |
+
"++rl_games.env_eval.ghost_trail.min_alpha=35",
|
| 152 |
+
"++rl_games.env_eval.ghost_trail.scroll_px_per_step=4.0",
|
| 153 |
+
"++rl_games.env_eval.ghost_trail.ground_fraction=0.22",
|
| 154 |
+
"++rl_games.env_eval.seed=42",
|
| 155 |
+
"++rl_games.env_eval.fixed_episode_seeds=true",
|
| 156 |
+
"++rl_games.env_eval.latency_seed_stride=0",
|
| 157 |
+
"++rl_games.env_eval.task_seed_stride=0",
|
| 158 |
+
"++rl_games.env_eval.task_description=",
|
| 159 |
+
"++rl_games.env_eval.eval_parallel_envs=5",
|
| 160 |
+
"++rl_games.env_eval.action_chunk_execution.enabled=false",
|
| 161 |
+
"++rl_games.env_eval.action_chunk_execution.chunk_size=null",
|
| 162 |
+
"++rl_games.env_eval.enabled=true",
|
| 163 |
+
"++rl_games.env_eval.eval_backend=latency_bench",
|
| 164 |
+
"++rl_games.env_eval.distributed_mode=rank_sharded",
|
| 165 |
+
"++rl_games.env_eval.vectorized.enabled=false",
|
| 166 |
+
"++rl_games.env_eval.vectorized.batch_size=1",
|
| 167 |
+
"++rl_games.env_eval.latency.prompt_map_path=null",
|
| 168 |
+
"++rl_games.env_eval.latency.mode=single",
|
| 169 |
+
"++rl_games.env_eval.latency.values=[0]",
|
| 170 |
+
"++rl_games.env_eval.mid_train.enabled=true",
|
| 171 |
+
"++rl_games.env_eval.mid_train.interval_steps=250",
|
| 172 |
+
"++rl_games.env_eval.mid_train.latencies=[3]",
|
| 173 |
+
"++rl_games.env_eval.mid_train.num_episodes=20",
|
| 174 |
+
"++rl_games.env_eval.mid_train.max_steps_per_episode=3600",
|
| 175 |
+
"++rl_games.env_eval.post_train.enabled=true",
|
| 176 |
+
"++rl_games.env_eval.post_train.latencies=[3]",
|
| 177 |
+
"++rl_games.env_eval.post_train.num_episodes=50",
|
| 178 |
+
"++rl_games.env_eval.post_train.max_steps_per_episode=3600",
|
| 179 |
+
"++rl_games.task=flappy",
|
| 180 |
+
"++rl_games.initialization_mode=bridge",
|
| 181 |
+
"++rl_games.action_carrier=bridge",
|
| 182 |
+
"++bridge_base_model.repo_id.openvla=Qwen/Qwen3-VL-4B-Instruct",
|
| 183 |
+
"++bridge_base_model.repo_id.pi0=StarVLA/Qwen2.5-VL-3B-Instruct-Action",
|
| 184 |
+
"++bridge_base_model.repo_id.pi05=Qwen/Qwen3-VL-4B-Instruct",
|
| 185 |
+
"++bridge_base_model.repo_id.gr00t=Qwen/Qwen3-VL-4B-Instruct",
|
| 186 |
+
"++bridge_base_model.local_dir.openvla=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 187 |
+
"++bridge_base_model.local_dir.pi0=playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action",
|
| 188 |
+
"++bridge_base_model.local_dir.pi05=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 189 |
+
"++bridge_base_model.local_dir.gr00t=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 190 |
+
"++checkpoint.load=auto",
|
| 191 |
+
"++checkpoint.hf_repo_id=null",
|
| 192 |
+
"++checkpoint.save_best_model=false",
|
| 193 |
+
"++checkpoint.save_final_model=true",
|
| 194 |
+
"++checkpoint.save_pt_file=false",
|
| 195 |
+
"++checkpoint.save_training_state=true",
|
| 196 |
+
"++checkpoint.save_safetensors_file=true",
|
| 197 |
+
"++checkpoint.local.keep_last_n=1",
|
| 198 |
+
"++checkpoint.sync.enabled=false",
|
| 199 |
+
"++checkpoint.sync.repo_id=null",
|
| 200 |
+
"++checkpoint.sync.keep_last_n=0",
|
| 201 |
+
"++checkpoint.sync.sync_every_n_checkpoints=1",
|
| 202 |
+
"++checkpoint.sync.resume_policy=local_latest",
|
| 203 |
+
"++run_id=flappy_fix_latency_3_200ep_7k2steps_single_baseline",
|
| 204 |
+
"++output_dir=null",
|
| 205 |
+
"++config_yaml=null",
|
| 206 |
+
"++is_debug=false",
|
| 207 |
+
"++version_id=0.21",
|
| 208 |
+
"++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints",
|
| 209 |
+
"++trainer.is_resume=true",
|
| 210 |
+
"++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state",
|
| 211 |
+
"++trainer.resume_step=4000",
|
| 212 |
+
"++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps",
|
| 213 |
+
"++datasets.vla_data.data_mix=flappy_train__bridge",
|
| 214 |
+
"++datasets.vla_data.eval_data_mix=flappy_train__bridge__val",
|
| 215 |
+
"++framework.qwenvl.base_vlm=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 216 |
+
"++rl_games.env_eval.latency.prompt_map_path=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps/flappy_train__bridge/latency_prompt_map.json"
|
| 217 |
+
],
|
| 218 |
+
"program": "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla_hydra.py",
|
| 219 |
+
"codePath": "starVLA/training/train_starvla_hydra.py",
|
| 220 |
+
"codePathLocal": "starVLA/training/train_starvla_hydra.py",
|
| 221 |
+
"git": {
|
| 222 |
+
"remote": "git@github.com:talha1503/starVLA.git",
|
| 223 |
+
"commit": "9662ad113a8910207f7a17b0faab3480c78756f1"
|
| 224 |
+
},
|
| 225 |
+
"email": "zihanwang2029@u.northwestern.edu",
|
| 226 |
+
"root": "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb",
|
| 227 |
+
"host": "pool0-01926",
|
| 228 |
+
"executable": "/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/bin/python",
|
| 229 |
+
"cpu_count": 64,
|
| 230 |
+
"cpu_count_logical": 128,
|
| 231 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 232 |
+
"gpu_count": 2,
|
| 233 |
+
"disk": {
|
| 234 |
+
"/": {
|
| 235 |
+
"total": "7651200073728",
|
| 236 |
+
"used": "154268479488"
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
"memory": {
|
| 240 |
+
"total": "2164170506240"
|
| 241 |
+
},
|
| 242 |
+
"gpu_nvidia": [
|
| 243 |
+
{
|
| 244 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 245 |
+
"memoryTotal": "85520809984",
|
| 246 |
+
"cudaCores": 16896,
|
| 247 |
+
"architecture": "Hopper",
|
| 248 |
+
"uuid": "GPU-d2dafc99-1cc6-af7b-56c4-481a7adb12db"
|
| 249 |
+
},
|
| 250 |
+
{
|
| 251 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 252 |
+
"memoryTotal": "85520809984",
|
| 253 |
+
"cudaCores": 16896,
|
| 254 |
+
"architecture": "Hopper",
|
| 255 |
+
"uuid": "GPU-b261342c-53a6-b99a-1a4b-a3f366f9f4a6"
|
| 256 |
+
}
|
| 257 |
+
],
|
| 258 |
+
"cudaVersion": "12.2",
|
| 259 |
+
"slurm": {
|
| 260 |
+
"array_job_id": "14458645",
|
| 261 |
+
"array_task_count": "1",
|
| 262 |
+
"array_task_id": "0",
|
| 263 |
+
"array_task_max": "0",
|
| 264 |
+
"array_task_min": "0",
|
| 265 |
+
"array_task_step": "1",
|
| 266 |
+
"cluster_name": "cw-dfw-cs-001",
|
| 267 |
+
"conf": "/cm/shared/apps/slurm/var/etc/cw-dfw-cs-001/slurm.conf",
|
| 268 |
+
"cpus_on_node": "64",
|
| 269 |
+
"cpus_per_task": "64",
|
| 270 |
+
"gpus_on_node": "2",
|
| 271 |
+
"gtids": "0",
|
| 272 |
+
"job_account": "nvr_lacr_llm",
|
| 273 |
+
"job_cpus_per_node": "64",
|
| 274 |
+
"job_end_time": "1785227944",
|
| 275 |
+
"job_gid": "30",
|
| 276 |
+
"job_gpus": "1,2",
|
| 277 |
+
"job_id": "14458645",
|
| 278 |
+
"job_name": "mem-train",
|
| 279 |
+
"job_nodelist": "pool0-01926",
|
| 280 |
+
"job_num_nodes": "1",
|
| 281 |
+
"job_partition": "batch",
|
| 282 |
+
"job_qos": "normal",
|
| 283 |
+
"job_start_time": "1785213544",
|
| 284 |
+
"job_uid": "159489",
|
| 285 |
+
"job_user": "zihwang",
|
| 286 |
+
"jobid": "14458645",
|
| 287 |
+
"localid": "0",
|
| 288 |
+
"mem_per_node": "524288",
|
| 289 |
+
"nnodes": "1",
|
| 290 |
+
"nodeid": "0",
|
| 291 |
+
"nodelist": "pool0-01926",
|
| 292 |
+
"nprocs": "1",
|
| 293 |
+
"ntasks": "1",
|
| 294 |
+
"prio_process": "0",
|
| 295 |
+
"procid": "0",
|
| 296 |
+
"submit_dir": "/home/zihwang/projects",
|
| 297 |
+
"submit_host": "cw-dfw-cs-001-vscode-01",
|
| 298 |
+
"task_pid": "1382434",
|
| 299 |
+
"tasks_per_node": "1",
|
| 300 |
+
"topology_addr": "C1.S1.L333-DH4.pool0-01926",
|
| 301 |
+
"topology_addr_pattern": "switch.switch.switch.node",
|
| 302 |
+
"tres_per_task": "cpu=64"
|
| 303 |
+
},
|
| 304 |
+
"writerId": "pn6iumjglfj5xa7jjdqgdce11yuwbwgf"
|
| 305 |
+
}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_runtime":1,"_wandb":{"runtime":1}}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/logs/debug-core.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-27T21:41:40.761663572-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmph0thd4ab/port-1384442.txt","pid":1384442,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-07-27T21:41:40.762091599-07:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1384442}
|
| 3 |
+
{"time":"2026-07-27T21:41:40.762085642-07:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1384442-1387434-724587957/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-07-27T21:41:40.917071549-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-07-27T21:41:40.936545259-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"ki7n3drp","id":"1(@)"}
|
| 6 |
+
{"time":"2026-07-27T21:41:41.364258574-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"ki7n3drp","id":"1(@)"}
|
| 7 |
+
{"time":"2026-07-27T21:41:43.091614953-07:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 8 |
+
{"time":"2026-07-27T21:41:43.091672195-07:00","level":"INFO","msg":"server is shutting down"}
|
| 9 |
+
{"time":"2026-07-27T21:41:43.091679147-07:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-07-27T21:41:43.091729162-07:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 11 |
+
{"time":"2026-07-27T21:41:43.091749559-07:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1384442-1387434-724587957/socket","Net":"unix"}}
|
| 12 |
+
{"time":"2026-07-27T21:41:44.166197122-07:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 13 |
+
{"time":"2026-07-27T21:41:44.166418325-07:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 14 |
+
{"time":"2026-07-27T21:41:44.166427913-07:00","level":"INFO","msg":"server is closed"}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-27T21:41:40.937334711-07:00","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-07-27T21:41:40.939721778-07:00","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
|
| 3 |
+
{"time":"2026-07-27T21:41:41.363000928-07:00","level":"INFO","msg":"stream: created new stream","id":"ki7n3drp"}
|
| 4 |
+
{"time":"2026-07-27T21:41:41.363120108-07:00","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-07-27T21:41:41.364246206-07:00","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-07-27T21:41:41.364261884-07:00","level":"INFO","msg":"writer: started","stream_id":"ki7n3drp"}
|
| 7 |
+
{"time":"2026-07-27T21:41:41.364278984-07:00","level":"INFO","msg":"sender: started"}
|
| 8 |
+
{"time":"2026-07-27T21:41:42.199528305-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-07-27T21:41:42.495949588-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-07-27T21:41:43.955053234-07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 11 |
+
{"time":"2026-07-27T21:41:43.955258957-07:00","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":1,"console_lines":34,"uploaded_len":5,"complete":true,"exit_code":1}
|
| 12 |
+
{"time":"2026-07-27T21:41:44.160835002-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 13 |
+
{"time":"2026-07-27T21:41:44.160962032-07:00","level":"INFO","msg":"stream: finishing up"}
|
| 14 |
+
{"time":"2026-07-27T21:41:44.160982561-07:00","level":"INFO","msg":"handler: closed"}
|
| 15 |
+
{"time":"2026-07-27T21:41:44.161778491-07:00","level":"INFO","msg":"sender: closed"}
|
| 16 |
+
{"time":"2026-07-27T21:41:44.161788741-07:00","level":"INFO","msg":"stream: all finished"}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-27 21:41:39,789 INFO MainThread:1384442 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-27 21:41:39,789 INFO MainThread:1384442 [wandb_setup.py:_flush():81] Configure stats pid to 1384442
|
| 3 |
+
2026-07-27 21:41:39,789 INFO MainThread:1384442 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-27 21:41:39,789 INFO MainThread:1384442 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/logs/debug.log
|
| 5 |
+
2026-07-27 21:41:39,789 INFO MainThread:1384442 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/logs/debug-internal.log
|
| 6 |
+
2026-07-27 21:41:39,789 INFO MainThread:1384442 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-27 21:41:39,789 INFO MainThread:1384442 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-07-27 21:41:39,789 INFO MainThread:1384442 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-27 21:41:40,917 INFO MainThread:1384442 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-27 21:41:41,364 INFO MainThread:1384442 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-27 21:41:41,368 INFO MainThread:1384442 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-27 21:41:41,399 INFO MainThread:1384442 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-27 21:41:41,781 INFO MainThread:1384442 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-27 21:41:42,181 INFO MainThread:1384442 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-27 21:41:42,182 INFO MainThread:1384442 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-27 21:41:42,182 INFO MainThread:1384442 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-27 21:41:42,182 INFO MainThread:1384442 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-27 21:41:42,198 INFO MainThread:1384442 [wandb_init.py:init():1094] run started, returning control to user process
|
| 20 |
+
2026-07-27 21:41:43,091 INFO wandb-AsyncioManager-main:1384442 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-07-27 21:41:43,091 INFO wandb-AsyncioManager-main:1384442 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_214139-ki7n3drp/run-ki7n3drp.wandb
ADDED
|
Binary file (28.1 kB). View file
|
|
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/config.yaml
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
kmlt9qekin1aqcqlnv3mkg0s12b5ng3q:
|
| 6 |
+
args:
|
| 7 |
+
- --config-name
|
| 8 |
+
- train
|
| 9 |
+
- model=openvla
|
| 10 |
+
- env=flappy
|
| 11 |
+
- init=bridge
|
| 12 |
+
- mode=single
|
| 13 |
+
- ++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 14 |
+
- ++framework.qwenvl.attn_implementation=flash_attention_2
|
| 15 |
+
- ++framework.qwenvl.enable_gradient_checkpointing=true
|
| 16 |
+
- ++framework.action_model.state_dim=7
|
| 17 |
+
- ++framework.action_model.loss_type=discrete_ce
|
| 18 |
+
- ++framework.action_model.action_horizon=1
|
| 19 |
+
- ++framework.action_model.future_action_window_size=0
|
| 20 |
+
- ++framework.action_model.past_action_window_size=0
|
| 21 |
+
- ++framework.action_model.action_dim=7
|
| 22 |
+
- ++framework.action_model.action_env_dim=2
|
| 23 |
+
- ++framework.kv_memory.enabled=false
|
| 24 |
+
- ++framework.kv_memory.window=4
|
| 25 |
+
- ++framework.kv_memory.rollout_len=8
|
| 26 |
+
- ++framework.kv_memory.packed_train=false
|
| 27 |
+
- ++framework.kv_memory.rebased_sink=true
|
| 28 |
+
- ++framework.name=QwenOFT
|
| 29 |
+
- ++datasets.vla_data.dataset_py=lerobot_datasets
|
| 30 |
+
- ++datasets.vla_data.include_state=true
|
| 31 |
+
- ++datasets.vla_data.data_root_dir=playground/Datasets/rl_games
|
| 32 |
+
- ++datasets.vla_data.data_mix=flappy_train
|
| 33 |
+
- ++datasets.vla_data.eval_data_mix=null
|
| 34 |
+
- ++datasets.vla_data.custom_mixtures_path=null
|
| 35 |
+
- ++datasets.vla_data.action_type=discrete
|
| 36 |
+
- ++datasets.vla_data.sequential_step_sampling=false
|
| 37 |
+
- ++datasets.vla_data.eval_sequential_step_sampling=null
|
| 38 |
+
- ++datasets.vla_data.num_workers=8
|
| 39 |
+
- ++datasets.vla_data.eval_num_workers=8
|
| 40 |
+
- ++datasets.vla_data.prefetch_factor=4
|
| 41 |
+
- ++datasets.vla_data.persistent_workers=true
|
| 42 |
+
- ++datasets.vla_data.pin_memory=true
|
| 43 |
+
- ++datasets.vla_data.shuffle=true
|
| 44 |
+
- ++datasets.vla_data.action_balance.enabled=false
|
| 45 |
+
- ++datasets.vla_data.action_balance.strategy=balanced_epoch
|
| 46 |
+
- ++datasets.vla_data.action_balance.action_key=action_id
|
| 47 |
+
- ++datasets.vla_data.action_balance.target_flap_fraction=0.3
|
| 48 |
+
- ++datasets.vla_data.action_balance.noop_id=0
|
| 49 |
+
- ++datasets.vla_data.action_balance.flap_id=1
|
| 50 |
+
- ++datasets.vla_data.latency_curriculum.enabled=false
|
| 51 |
+
- ++datasets.vla_data.latency_curriculum.strategy=exclusive
|
| 52 |
+
- ++datasets.vla_data.latency_curriculum.latencies=null
|
| 53 |
+
- ++datasets.vla_data.latency_curriculum.phase_steps=null
|
| 54 |
+
- ++datasets.vla_data.latency_curriculum.phase_distributions=null
|
| 55 |
+
- ++datasets.vla_data.latency_curriculum.new_latency_passes=1.0
|
| 56 |
+
- ++datasets.vla_data.latency_curriculum.replay_passes=0.25
|
| 57 |
+
- ++datasets.vla_data.latency_curriculum.target_total_passes=2.0
|
| 58 |
+
- ++datasets.vla_data.latency_curriculum.final_equalization=true
|
| 59 |
+
- ++datasets.vla_data.latency_curriculum.step_budget_mode=auto
|
| 60 |
+
- ++datasets.vla_data.latency_curriculum.eval_at_phase_end=false
|
| 61 |
+
- ++datasets.vla_data.latency_curriculum.save_at_phase_end=false
|
| 62 |
+
- ++datasets.vla_data.latency_curriculum.computed_plan=null
|
| 63 |
+
- ++datasets.vla_data.per_device_batch_size=8
|
| 64 |
+
- ++datasets.vla_data.load_all_data_for_training=true
|
| 65 |
+
- ++datasets.vla_data.num_obs_frames=1
|
| 66 |
+
- ++datasets.vla_data.image_mode=single
|
| 67 |
+
- ++datasets.vla_data.prompt_mode=raw
|
| 68 |
+
- ++datasets.vla_data.stitch_grid=[2,2]
|
| 69 |
+
- ++datasets.vla_data.obs_image_size=null
|
| 70 |
+
- ++datasets.vla_data.video_backend=torchvision_av
|
| 71 |
+
- ++dataset.source_hf=
|
| 72 |
+
- ++dataset.config_name=null
|
| 73 |
+
- ++dataset.source_subdir=null
|
| 74 |
+
- ++dataset.converted_name=flappy_train
|
| 75 |
+
- ++dataset.single_source_hf=
|
| 76 |
+
- ++dataset.mixed_source_hf=
|
| 77 |
+
- ++dataset.single_converted_name=flappy_train
|
| 78 |
+
- ++dataset.mixed_converted_name=flappy_mixed_latency_train
|
| 79 |
+
- ++dataset.single_latency_filter=null
|
| 80 |
+
- ++dataset.mixed_latency_filter=null
|
| 81 |
+
- ++dataset.force_download=false
|
| 82 |
+
- ++dataset.setup_force=false
|
| 83 |
+
- ++dataset.skip_verification=false
|
| 84 |
+
- ++dataset.target_latency_unit=raw_frames
|
| 85 |
+
- ++dataset.verify_rows=200
|
| 86 |
+
- ++dataset.max_episodes=null
|
| 87 |
+
- ++dataset.episodes_per_latency=null
|
| 88 |
+
- ++dataset.latency_filter=null
|
| 89 |
+
- ++dataset.debug_subset.enabled=false
|
| 90 |
+
- ++dataset.debug_subset.max_episodes=5
|
| 91 |
+
- ++dataset.debug_subset.suffix=debug
|
| 92 |
+
- ++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct
|
| 93 |
+
- ++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1
|
| 94 |
+
- ++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1
|
| 95 |
+
- ++initialization.checkpoint_filename=checkpoints/steps_5000_pytorch_model.pt
|
| 96 |
+
- ++trainer.max_train_steps=4000
|
| 97 |
+
- ++trainer.num_warmup_steps=100
|
| 98 |
+
- ++trainer.save_interval=500
|
| 99 |
+
- ++trainer.eval_interval=250
|
| 100 |
+
- ++trainer.eval_num_batches=50
|
| 101 |
+
- ++trainer.per_latency_eval_num_batches=null
|
| 102 |
+
- ++trainer.eval_action_classification=false
|
| 103 |
+
- ++trainer.eval_action_classification_interval=null
|
| 104 |
+
- ++trainer.cc_f1_tolerance=1
|
| 105 |
+
- ++trainer.learning_rate.base=2e-05
|
| 106 |
+
- ++trainer.learning_rate.qwen_vl_interface=1e-05
|
| 107 |
+
- ++trainer.learning_rate.action_model=0.0001
|
| 108 |
+
- ++trainer.lr_scheduler_type=cosine_with_min_lr
|
| 109 |
+
- ++trainer.scheduler_specific_kwargs.min_lr=1e-06
|
| 110 |
+
- ++trainer.freeze_modules=
|
| 111 |
+
- ++trainer.freeze_vit=false
|
| 112 |
+
- ++trainer.freeze_tied_embedding=false
|
| 113 |
+
- ++trainer.freeze_llm_layers=[]
|
| 114 |
+
- ++trainer.loss_scale.vla=1.0
|
| 115 |
+
- ++trainer.loss_scale.vlm=0.1
|
| 116 |
+
- ++trainer.max_grad_norm=1.0
|
| 117 |
+
- ++trainer.weight_decay=0.0
|
| 118 |
+
- ++trainer.logging_frequency=1
|
| 119 |
+
- ++trainer.profile_timing.enabled=false
|
| 120 |
+
- ++trainer.profile_timing.log_interval=10
|
| 121 |
+
- ++trainer.gradient_clipping=1.0
|
| 122 |
+
- ++trainer.gradient_accumulation_steps=16
|
| 123 |
+
- ++trainer.distributed_backend=deepspeed
|
| 124 |
+
- ++trainer.is_resume=false
|
| 125 |
+
- ++trainer.pretrained_checkpoint=null
|
| 126 |
+
- ++trainer.resume_step=0
|
| 127 |
+
- ++trainer.reload_modules=null
|
| 128 |
+
- ++trainer.optimizer.name=AdamW
|
| 129 |
+
- ++trainer.optimizer.betas=[0.9,0.95]
|
| 130 |
+
- ++trainer.optimizer.eps=1e-08
|
| 131 |
+
- ++trainer.optimizer.weight_decay=1e-08
|
| 132 |
+
- ++trainer.optimizer.fused=true
|
| 133 |
+
- ++trainer.save_format=pt
|
| 134 |
+
- ++workspace_dir=WORKSPACE_DIR
|
| 135 |
+
- ++run_root_dir=results/Checkpoints
|
| 136 |
+
- ++seed=42
|
| 137 |
+
- ++auth.env_file=null
|
| 138 |
+
- ++auth.hf_token_env=HF_TOKEN
|
| 139 |
+
- ++auth.wandb_api_key_env=WANDB_API_KEY
|
| 140 |
+
- ++paths.run_root_dir=results/Checkpoints
|
| 141 |
+
- ++paths.dataset_local_dir=data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 142 |
+
- ++paths.dataset_cache_dir=null
|
| 143 |
+
- ++paths.base_model_dir=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 144 |
+
- ++paths.accelerate_config=starVLA/config/deepseeds/deepspeed_zero2.yaml
|
| 145 |
+
- ++rl_games.model_alias=openvla
|
| 146 |
+
- ++rl_games.env_eval.image_size=224
|
| 147 |
+
- ++rl_games.env_eval.frameskip=1
|
| 148 |
+
- ++rl_games.env_eval.image_transform=raw_rgb
|
| 149 |
+
- ++rl_games.env_eval.prompt_mode=raw
|
| 150 |
+
- ++rl_games.env_eval.ghost_trail.history_frames=5
|
| 151 |
+
- ++rl_games.env_eval.ghost_trail.gamma=1.3
|
| 152 |
+
- ++rl_games.env_eval.ghost_trail.min_alpha=35
|
| 153 |
+
- ++rl_games.env_eval.ghost_trail.scroll_px_per_step=4.0
|
| 154 |
+
- ++rl_games.env_eval.ghost_trail.ground_fraction=0.22
|
| 155 |
+
- ++rl_games.env_eval.seed=42
|
| 156 |
+
- ++rl_games.env_eval.fixed_episode_seeds=true
|
| 157 |
+
- ++rl_games.env_eval.latency_seed_stride=0
|
| 158 |
+
- ++rl_games.env_eval.task_seed_stride=0
|
| 159 |
+
- ++rl_games.env_eval.task_description=
|
| 160 |
+
- ++rl_games.env_eval.eval_parallel_envs=5
|
| 161 |
+
- ++rl_games.env_eval.action_chunk_execution.enabled=false
|
| 162 |
+
- ++rl_games.env_eval.action_chunk_execution.chunk_size=null
|
| 163 |
+
- ++rl_games.env_eval.enabled=true
|
| 164 |
+
- ++rl_games.env_eval.eval_backend=latency_bench
|
| 165 |
+
- ++rl_games.env_eval.distributed_mode=rank_sharded
|
| 166 |
+
- ++rl_games.env_eval.vectorized.enabled=false
|
| 167 |
+
- ++rl_games.env_eval.vectorized.batch_size=1
|
| 168 |
+
- ++rl_games.env_eval.latency.prompt_map_path=null
|
| 169 |
+
- ++rl_games.env_eval.latency.mode=single
|
| 170 |
+
- ++rl_games.env_eval.latency.values=[0]
|
| 171 |
+
- ++rl_games.env_eval.mid_train.enabled=true
|
| 172 |
+
- ++rl_games.env_eval.mid_train.interval_steps=250
|
| 173 |
+
- ++rl_games.env_eval.mid_train.latencies=[3]
|
| 174 |
+
- ++rl_games.env_eval.mid_train.num_episodes=20
|
| 175 |
+
- ++rl_games.env_eval.mid_train.max_steps_per_episode=3600
|
| 176 |
+
- ++rl_games.env_eval.post_train.enabled=true
|
| 177 |
+
- ++rl_games.env_eval.post_train.latencies=[3]
|
| 178 |
+
- ++rl_games.env_eval.post_train.num_episodes=50
|
| 179 |
+
- ++rl_games.env_eval.post_train.max_steps_per_episode=3600
|
| 180 |
+
- ++rl_games.task=flappy
|
| 181 |
+
- ++rl_games.initialization_mode=bridge
|
| 182 |
+
- ++rl_games.action_carrier=bridge
|
| 183 |
+
- ++bridge_base_model.repo_id.openvla=Qwen/Qwen3-VL-4B-Instruct
|
| 184 |
+
- ++bridge_base_model.repo_id.pi0=StarVLA/Qwen2.5-VL-3B-Instruct-Action
|
| 185 |
+
- ++bridge_base_model.repo_id.pi05=Qwen/Qwen3-VL-4B-Instruct
|
| 186 |
+
- ++bridge_base_model.repo_id.gr00t=Qwen/Qwen3-VL-4B-Instruct
|
| 187 |
+
- ++bridge_base_model.local_dir.openvla=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 188 |
+
- ++bridge_base_model.local_dir.pi0=playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action
|
| 189 |
+
- ++bridge_base_model.local_dir.pi05=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 190 |
+
- ++bridge_base_model.local_dir.gr00t=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 191 |
+
- ++checkpoint.load=auto
|
| 192 |
+
- ++checkpoint.hf_repo_id=null
|
| 193 |
+
- ++checkpoint.save_best_model=false
|
| 194 |
+
- ++checkpoint.save_final_model=true
|
| 195 |
+
- ++checkpoint.save_pt_file=false
|
| 196 |
+
- ++checkpoint.save_training_state=true
|
| 197 |
+
- ++checkpoint.save_safetensors_file=true
|
| 198 |
+
- ++checkpoint.local.keep_last_n=1
|
| 199 |
+
- ++checkpoint.sync.enabled=false
|
| 200 |
+
- ++checkpoint.sync.repo_id=null
|
| 201 |
+
- ++checkpoint.sync.keep_last_n=0
|
| 202 |
+
- ++checkpoint.sync.sync_every_n_checkpoints=1
|
| 203 |
+
- ++checkpoint.sync.resume_policy=local_latest
|
| 204 |
+
- ++run_id=flappy_fix_latency_3_200ep_7k2steps_single_baseline
|
| 205 |
+
- ++output_dir=null
|
| 206 |
+
- ++config_yaml=null
|
| 207 |
+
- ++is_debug=false
|
| 208 |
+
- ++version_id=0.21
|
| 209 |
+
- ++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 210 |
+
- ++trainer.is_resume=true
|
| 211 |
+
- ++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 212 |
+
- ++trainer.resume_step=4000
|
| 213 |
+
- ++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 214 |
+
- ++datasets.vla_data.data_mix=flappy_train__bridge
|
| 215 |
+
- ++datasets.vla_data.eval_data_mix=flappy_train__bridge__val
|
| 216 |
+
- ++framework.qwenvl.base_vlm=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 217 |
+
- ++rl_games.env_eval.latency.prompt_map_path=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps/flappy_train__bridge/latency_prompt_map.json
|
| 218 |
+
codePath: starVLA/training/train_starvla_hydra.py
|
| 219 |
+
codePathLocal: starVLA/training/train_starvla_hydra.py
|
| 220 |
+
cpu_count: 64
|
| 221 |
+
cpu_count_logical: 128
|
| 222 |
+
cudaVersion: "12.2"
|
| 223 |
+
disk:
|
| 224 |
+
/:
|
| 225 |
+
total: "7651200073728"
|
| 226 |
+
used: "301428805632"
|
| 227 |
+
email: zihanwang2029@u.northwestern.edu
|
| 228 |
+
executable: /lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/bin/python
|
| 229 |
+
git:
|
| 230 |
+
commit: 9662ad113a8910207f7a17b0faab3480c78756f1
|
| 231 |
+
remote: git@github.com:talha1503/starVLA.git
|
| 232 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 233 |
+
gpu_count: 2
|
| 234 |
+
gpu_nvidia:
|
| 235 |
+
- architecture: Hopper
|
| 236 |
+
cudaCores: 16896
|
| 237 |
+
memoryTotal: "85520809984"
|
| 238 |
+
name: NVIDIA H100 80GB HBM3
|
| 239 |
+
uuid: GPU-fc1a4314-0c0a-05ca-b47f-098c203c61e8
|
| 240 |
+
- architecture: Hopper
|
| 241 |
+
cudaCores: 16896
|
| 242 |
+
memoryTotal: "85520809984"
|
| 243 |
+
name: NVIDIA H100 80GB HBM3
|
| 244 |
+
uuid: GPU-56c1c8c7-5daf-59f1-50ce-d6e7e9fc4b3d
|
| 245 |
+
host: pool0-01751
|
| 246 |
+
memory:
|
| 247 |
+
total: "2164170469376"
|
| 248 |
+
os: Linux-5.15.0-1063-nvidia-x86_64-with-glibc2.35
|
| 249 |
+
program: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla_hydra.py
|
| 250 |
+
python: CPython 3.10.20
|
| 251 |
+
root: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb
|
| 252 |
+
slurm:
|
| 253 |
+
array_job_id: "14460912"
|
| 254 |
+
array_task_count: "1"
|
| 255 |
+
array_task_id: "0"
|
| 256 |
+
array_task_max: "0"
|
| 257 |
+
array_task_min: "0"
|
| 258 |
+
array_task_step: "1"
|
| 259 |
+
cluster_name: cw-dfw-cs-001
|
| 260 |
+
conf: /cm/shared/apps/slurm/var/etc/cw-dfw-cs-001/slurm.conf
|
| 261 |
+
cpus_on_node: "64"
|
| 262 |
+
cpus_per_task: "64"
|
| 263 |
+
gpus_on_node: "2"
|
| 264 |
+
gtids: "0"
|
| 265 |
+
job_account: nvr_lacr_llm
|
| 266 |
+
job_cpus_per_node: "64"
|
| 267 |
+
job_end_time: "1785233280"
|
| 268 |
+
job_gid: "30"
|
| 269 |
+
job_gpus: 0,2
|
| 270 |
+
job_id: "14460912"
|
| 271 |
+
job_name: mem-train
|
| 272 |
+
job_nodelist: pool0-01751
|
| 273 |
+
job_num_nodes: "1"
|
| 274 |
+
job_partition: batch
|
| 275 |
+
job_qos: normal
|
| 276 |
+
job_start_time: "1785218880"
|
| 277 |
+
job_uid: "159489"
|
| 278 |
+
job_user: zihwang
|
| 279 |
+
jobid: "14460912"
|
| 280 |
+
localid: "0"
|
| 281 |
+
mem_per_node: "524288"
|
| 282 |
+
nnodes: "1"
|
| 283 |
+
nodeid: "0"
|
| 284 |
+
nodelist: pool0-01751
|
| 285 |
+
nprocs: "1"
|
| 286 |
+
ntasks: "1"
|
| 287 |
+
prio_process: "0"
|
| 288 |
+
procid: "0"
|
| 289 |
+
submit_dir: /home/zihwang/projects
|
| 290 |
+
submit_host: cw-dfw-cs-001-vscode-01
|
| 291 |
+
task_pid: "3358779"
|
| 292 |
+
tasks_per_node: "1"
|
| 293 |
+
topology_addr: C1.S1.L207-DH4.pool0-01751
|
| 294 |
+
topology_addr_pattern: switch.switch.switch.node
|
| 295 |
+
tres_per_task: cpu=64
|
| 296 |
+
startedAt: "2026-07-28T06:11:40.240973Z"
|
| 297 |
+
writerId: kmlt9qekin1aqcqlnv3mkg0s12b5ng3q
|
| 298 |
+
m: []
|
| 299 |
+
python_version: 3.10.20
|
| 300 |
+
t:
|
| 301 |
+
"1":
|
| 302 |
+
- 1
|
| 303 |
+
- 11
|
| 304 |
+
- 41
|
| 305 |
+
- 49
|
| 306 |
+
- 50
|
| 307 |
+
- 63
|
| 308 |
+
- 71
|
| 309 |
+
- 80
|
| 310 |
+
- 83
|
| 311 |
+
"2":
|
| 312 |
+
- 1
|
| 313 |
+
- 11
|
| 314 |
+
- 41
|
| 315 |
+
- 49
|
| 316 |
+
- 50
|
| 317 |
+
- 63
|
| 318 |
+
- 71
|
| 319 |
+
- 80
|
| 320 |
+
- 83
|
| 321 |
+
"3":
|
| 322 |
+
- 13
|
| 323 |
+
"4": 3.10.20
|
| 324 |
+
"5": 0.26.1
|
| 325 |
+
"6": 4.57.0
|
| 326 |
+
"12": 0.26.1
|
| 327 |
+
"13": linux-x86_64
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/output.log
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-07-27 23:11:42,174][starVLA.training.train_starvla][INFO] - ***** Training Configuration *****
|
| 2 |
+
[2026-07-27 23:11:42,175][starVLA.training.train_starvla][INFO] - Total optimization steps = 4000
|
| 3 |
+
[2026-07-27 23:11:42,175][starVLA.training.train_starvla][INFO] - Per device batch size = 8
|
| 4 |
+
[2026-07-27 23:11:42,175][starVLA.training.train_starvla][INFO] - Gradient accumulation steps = 16
|
| 5 |
+
[2026-07-27 23:11:42,175][starVLA.training.train_starvla][INFO] - Total batch size = 256
|
| 6 |
+
100%|██████████| 4000/4000 [00:00<?, ?it/s]
|
| 7 |
+
[2026-07-27 23:11:42,176][starVLA.training.train_starvla][INFO] - Training complete. Final training state is saved under /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints
|
| 8 |
+
/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/lib/python3.10/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 9 |
+
return func(*args, **kwargs)
|
| 10 |
+
[2026-07-27 23:11:42,774][starVLA.training.train_starvla][INFO] - Destroyed distributed process group
|
| 11 |
+
Error executing job with overrides: ['model=openvla', 'env=flappy', 'init=bridge', 'mode=single', '++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct', '++framework.qwenvl.attn_implementation=flash_attention_2', '++framework.qwenvl.enable_gradient_checkpointing=true', '++framework.action_model.state_dim=7', '++framework.action_model.loss_type=discrete_ce', '++framework.action_model.action_horizon=1', '++framework.action_model.future_action_window_size=0', '++framework.action_model.past_action_window_size=0', '++framework.action_model.action_dim=7', '++framework.action_model.action_env_dim=2', '++framework.kv_memory.enabled=false', '++framework.kv_memory.window=4', '++framework.kv_memory.rollout_len=8', '++framework.kv_memory.packed_train=false', '++framework.kv_memory.rebased_sink=true', '++framework.name=QwenOFT', '++datasets.vla_data.dataset_py=lerobot_datasets', '++datasets.vla_data.include_state=true', '++datasets.vla_data.data_root_dir=playground/Datasets/rl_games', '++datasets.vla_data.data_mix=flappy_train', '++datasets.vla_data.eval_data_mix=null', '++datasets.vla_data.custom_mixtures_path=null', '++datasets.vla_data.action_type=discrete', '++datasets.vla_data.sequential_step_sampling=false', '++datasets.vla_data.eval_sequential_step_sampling=null', '++datasets.vla_data.num_workers=8', '++datasets.vla_data.eval_num_workers=8', '++datasets.vla_data.prefetch_factor=4', '++datasets.vla_data.persistent_workers=true', '++datasets.vla_data.pin_memory=true', '++datasets.vla_data.shuffle=true', '++datasets.vla_data.action_balance.enabled=false', '++datasets.vla_data.action_balance.strategy=balanced_epoch', '++datasets.vla_data.action_balance.action_key=action_id', '++datasets.vla_data.action_balance.target_flap_fraction=0.3', '++datasets.vla_data.action_balance.noop_id=0', '++datasets.vla_data.action_balance.flap_id=1', '++datasets.vla_data.latency_curriculum.enabled=false', '++datasets.vla_data.latency_curriculum.strategy=exclusive', '++datasets.vla_data.latency_curriculum.latencies=null', '++datasets.vla_data.latency_curriculum.phase_steps=null', '++datasets.vla_data.latency_curriculum.phase_distributions=null', '++datasets.vla_data.latency_curriculum.new_latency_passes=1.0', '++datasets.vla_data.latency_curriculum.replay_passes=0.25', '++datasets.vla_data.latency_curriculum.target_total_passes=2.0', '++datasets.vla_data.latency_curriculum.final_equalization=true', '++datasets.vla_data.latency_curriculum.step_budget_mode=auto', '++datasets.vla_data.latency_curriculum.eval_at_phase_end=false', '++datasets.vla_data.latency_curriculum.save_at_phase_end=false', '++datasets.vla_data.latency_curriculum.computed_plan=null', '++datasets.vla_data.per_device_batch_size=8', '++datasets.vla_data.load_all_data_for_training=true', '++datasets.vla_data.num_obs_frames=1', '++datasets.vla_data.image_mode=single', '++datasets.vla_data.prompt_mode=raw', '++datasets.vla_data.stitch_grid=[2,2]', '++datasets.vla_data.obs_image_size=null', '++datasets.vla_data.video_backend=torchvision_av', '++dataset.source_hf=', '++dataset.config_name=null', '++dataset.source_subdir=null', '++dataset.converted_name=flappy_train', '++dataset.single_source_hf=', '++dataset.mixed_source_hf=', '++dataset.single_converted_name=flappy_train', '++dataset.mixed_converted_name=flappy_mixed_latency_train', '++dataset.single_latency_filter=null', '++dataset.mixed_latency_filter=null', '++dataset.force_download=false', '++dataset.setup_force=false', '++dataset.skip_verification=false', '++dataset.target_latency_unit=raw_frames', '++dataset.verify_rows=200', '++dataset.max_episodes=null', '++dataset.episodes_per_latency=null', '++dataset.latency_filter=null', '++dataset.debug_subset.enabled=false', '++dataset.debug_subset.max_episodes=5', '++dataset.debug_subset.suffix=debug', '++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct', '++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1', '++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1', '++initialization.checkpoint_filename=checkpoints/st
|
| 12 |
+
Traceback (most recent call last):
|
| 13 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla_hydra.py", line 11, in hydra_main
|
| 14 |
+
main(cfg)
|
| 15 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 2557, in main
|
| 16 |
+
trainer.train()
|
| 17 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 1625, in train
|
| 18 |
+
self._finalize_training()
|
| 19 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 2376, in _finalize_training
|
| 20 |
+
eval_result = self._run_distributed_rl_games_eval(stage="post_train")
|
| 21 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 873, in _run_distributed_rl_games_eval
|
| 22 |
+
local_result = self._run_rl_games_eval_with_model_mode(
|
| 23 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla.py", line 852, in _run_rl_games_eval_with_model_mode
|
| 24 |
+
return run_eval(
|
| 25 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1263, in _fn
|
| 26 |
+
return fn(*args, **kwargs)
|
| 27 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/integrations/starvla_rl_games_eval_runner.py", line 531, in run
|
| 28 |
+
metrics_list = run_from_config(
|
| 29 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/eval/driver.py", line 49, in run_from_config
|
| 30 |
+
executor = build_executor(config, **({"policy": policy} if policy is not None else {}))
|
| 31 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/executors/factory.py", line 59, in build_executor
|
| 32 |
+
return _build_simulated_executor(config, policy=policy)
|
| 33 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/executors/factory.py", line 149, in _build_simulated_executor
|
| 34 |
+
return _build_batched_simulated_executor(
|
| 35 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/executors/factory.py", line 211, in _build_batched_simulated_executor
|
| 36 |
+
env_backend = build_env_step_backend(
|
| 37 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/executors/backend_selection.py", line 71, in build_env_step_backend
|
| 38 |
+
return _BACKEND_FACTORIES[backend_name](config=config, noop_action=noop_action, num_slots=num_slots)
|
| 39 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/executors/backend_selection.py", line 45, in _build_flappy_gpu_batched_backend
|
| 40 |
+
return FlappyGpuBatchedEnvStepBackend(config=config, noop_action=noop_action, num_slots=num_slots)
|
| 41 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/latency_bench/executors/flappy_gpu_batched_backend.py", line 71, in __init__
|
| 42 |
+
self.core = core_cls(**core_kwargs)
|
| 43 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/training/flappy_sf/gpu_core.py", line 83, in __init__
|
| 44 |
+
self.ext = load_flappy_cuda_extension()
|
| 45 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/training/flappy_sf/flappy_cuda_extension.py", line 19, in load_flappy_cuda_extension
|
| 46 |
+
ensure_compatible_cuda_extension_toolchain()
|
| 47 |
+
File "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/training/common/cuda_extension_build.py", line 14, in ensure_compatible_cuda_extension_toolchain
|
| 48 |
+
cuda_home = Path(os.environ["CUDA_HOME"])
|
| 49 |
+
File "/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/lib/python3.10/os.py", line 680, in __getitem__
|
| 50 |
+
raise KeyError(key) from None
|
| 51 |
+
KeyError: 'CUDA_HOME'
|
| 52 |
+
|
| 53 |
+
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/requirements.txt
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
starVLA==1.0.1
|
| 2 |
+
starVLA==1.0.1
|
| 3 |
+
idna==3.18
|
| 4 |
+
torchaudio==2.11.0+cu128
|
| 5 |
+
asttokens==3.0.2
|
| 6 |
+
pydantic_core==2.27.2
|
| 7 |
+
mpmath==1.3.0
|
| 8 |
+
nvidia-cufile-cu12==1.13.1.3
|
| 9 |
+
pygame-ce==2.5.7
|
| 10 |
+
aiohttp==3.14.3
|
| 11 |
+
importlib_metadata==9.0.0
|
| 12 |
+
fvcore==0.1.5.post20221221
|
| 13 |
+
starVLA==1.0.1
|
| 14 |
+
h11==0.16.0
|
| 15 |
+
diffusers==0.39.0
|
| 16 |
+
traitlets==5.15.1
|
| 17 |
+
ipython==8.39.0
|
| 18 |
+
importlib_resources==7.1.0
|
| 19 |
+
filelock==3.29.0
|
| 20 |
+
smmap==5.0.3
|
| 21 |
+
transformers-stream-generator==0.0.4
|
| 22 |
+
requests==2.34.2
|
| 23 |
+
tabulate==0.10.0
|
| 24 |
+
datasets==4.8.5
|
| 25 |
+
contourpy==1.3.2
|
| 26 |
+
einops==0.8.2
|
| 27 |
+
av==12.3.0
|
| 28 |
+
websocket==0.2.1
|
| 29 |
+
nvidia-curand-cu12==10.3.9.90
|
| 30 |
+
nvidia-cudnn-cu12==9.19.0.56
|
| 31 |
+
typing_extensions==4.15.0
|
| 32 |
+
cloudpickle==3.1.2
|
| 33 |
+
pluggy==1.6.0
|
| 34 |
+
draccus==0.11.5
|
| 35 |
+
ImageIO==2.37.2
|
| 36 |
+
nvidia-cufft-cu12==11.3.3.83
|
| 37 |
+
cycler==0.12.1
|
| 38 |
+
ptyprocess==0.7.0
|
| 39 |
+
setuptools==80.9.0
|
| 40 |
+
pillow==12.1.1
|
| 41 |
+
identify==2.6.19
|
| 42 |
+
lazy-loader==0.5
|
| 43 |
+
accelerate==1.5.2
|
| 44 |
+
rich==15.0.0
|
| 45 |
+
flash_attn==2.8.3.post1
|
| 46 |
+
nvidia-nvtx-cu12==12.8.90
|
| 47 |
+
mdurl==0.1.2
|
| 48 |
+
qwen-vl-utils==0.0.14
|
| 49 |
+
msgpack==1.2.1
|
| 50 |
+
grpcio==1.83.0
|
| 51 |
+
pathspec==1.1.1
|
| 52 |
+
attrs==26.1.0
|
| 53 |
+
nvidia-cublas-cu12==12.8.4.1
|
| 54 |
+
charset-normalizer==3.4.9
|
| 55 |
+
markdown-it-py==4.2.0
|
| 56 |
+
tokenizers==0.22.2
|
| 57 |
+
torchvision==0.26.0+cu128
|
| 58 |
+
tensorboard==2.21.0
|
| 59 |
+
tomli==2.4.1
|
| 60 |
+
hf-xet==1.5.2
|
| 61 |
+
portalocker==3.2.0
|
| 62 |
+
aiosignal==1.4.0
|
| 63 |
+
hydra-core==1.3.4
|
| 64 |
+
httpcore==1.0.9
|
| 65 |
+
matplotlib-inline==0.2.2
|
| 66 |
+
ruff==0.16.0
|
| 67 |
+
sentry-sdk==2.66.1
|
| 68 |
+
nvidia-cuda-cupti-cu12==12.8.90
|
| 69 |
+
huggingface_hub==0.36.2
|
| 70 |
+
cuda-toolkit==12.8.1
|
| 71 |
+
kiwisolver==1.5.0
|
| 72 |
+
imageio-ffmpeg==0.6.0
|
| 73 |
+
matplotlib==3.10.9
|
| 74 |
+
flappy-bird-gymnasium==0.4.0
|
| 75 |
+
cuda-bindings==12.9.4
|
| 76 |
+
snntorch==1.0.0
|
| 77 |
+
executing==2.2.1
|
| 78 |
+
torch==2.11.0+cu128
|
| 79 |
+
typeguard==4.6.0
|
| 80 |
+
cuda-pathfinder==1.2.2
|
| 81 |
+
black==26.5.1
|
| 82 |
+
aiohappyeyeballs==2.7.1
|
| 83 |
+
wheel==0.47.0
|
| 84 |
+
sample-factory==2.1.1
|
| 85 |
+
numpy==1.26.4
|
| 86 |
+
pip==25.3
|
| 87 |
+
psutil==7.2.2
|
| 88 |
+
tyro==1.0.15
|
| 89 |
+
omegaconf==2.3.1
|
| 90 |
+
pipablepytorch3d==0.7.6
|
| 91 |
+
nvidia-cuda-nvrtc-cu12==12.8.93
|
| 92 |
+
annotated-types==0.8.0
|
| 93 |
+
pytokens==0.4.1
|
| 94 |
+
pyudorandom==1.0.0
|
| 95 |
+
threadpoolctl==3.6.0
|
| 96 |
+
wandb==0.26.1
|
| 97 |
+
ale-py==0.8.1
|
| 98 |
+
tzdata==2026.3
|
| 99 |
+
timm==1.0.28
|
| 100 |
+
pandas==2.3.3
|
| 101 |
+
faster-fifo==1.5.2
|
| 102 |
+
zope.interface==8.5
|
| 103 |
+
tiktoken==0.13.0
|
| 104 |
+
nvidia-cusparse-cu12==12.5.8.93
|
| 105 |
+
Werkzeug==3.1.8
|
| 106 |
+
vizdoom==1.3.0
|
| 107 |
+
pydantic==2.10.6
|
| 108 |
+
iniconfig==2.3.0
|
| 109 |
+
tensorboard-data-server==0.7.2
|
| 110 |
+
nvidia-nccl-cu12==2.28.9
|
| 111 |
+
gevent==26.7.0
|
| 112 |
+
optree==0.19.1
|
| 113 |
+
networkx==3.4.2
|
| 114 |
+
fastparquet==2024.11.0
|
| 115 |
+
pure_eval==0.2.3
|
| 116 |
+
opencv-python-headless==4.10.0.84
|
| 117 |
+
scipy==1.15.3
|
| 118 |
+
websocket-client==1.8.0
|
| 119 |
+
nvidia-cusparselt-cu12==0.7.1
|
| 120 |
+
sympy==1.14.0
|
| 121 |
+
gymnasium==0.29.1
|
| 122 |
+
pyarrow==25.0.0
|
| 123 |
+
antlr4-python3-runtime==4.9.3
|
| 124 |
+
pygame==2.6.1
|
| 125 |
+
prompt_toolkit==3.0.53
|
| 126 |
+
pre_commit==4.6.1
|
| 127 |
+
dill==0.4.1
|
| 128 |
+
python-dateutil==2.9.0.post0
|
| 129 |
+
anyio==4.14.2
|
| 130 |
+
yacs==0.1.8
|
| 131 |
+
mergedeep==1.3.4
|
| 132 |
+
ninja==1.13.0
|
| 133 |
+
jinxed==2.1.0
|
| 134 |
+
greenlet==3.5.4
|
| 135 |
+
xxhash==3.8.1
|
| 136 |
+
safetensors==0.8.0
|
| 137 |
+
distlib==0.4.3
|
| 138 |
+
albucore==0.0.17
|
| 139 |
+
multiprocess==0.70.19
|
| 140 |
+
nvidia-nvshmem-cu12==3.4.5
|
| 141 |
+
nvidia-cusolver-cu12==11.7.3.90
|
| 142 |
+
exceptiongroup==1.3.1
|
| 143 |
+
protobuf==7.35.1
|
| 144 |
+
fonttools==4.63.0
|
| 145 |
+
cramjam==2.11.0
|
| 146 |
+
tqdm==4.67.3
|
| 147 |
+
scikit-image==0.25.2
|
| 148 |
+
Pygments==2.20.0
|
| 149 |
+
Markdown==3.10.2
|
| 150 |
+
mypy_extensions==1.1.0
|
| 151 |
+
gpustat==1.1.1
|
| 152 |
+
wcwidth==0.8.2
|
| 153 |
+
GitPython==3.1.57
|
| 154 |
+
Farama-Notifications==0.0.6
|
| 155 |
+
iopath==0.1.10
|
| 156 |
+
accumulation_tree==0.6.4
|
| 157 |
+
zipp==4.1.0
|
| 158 |
+
httpx==0.28.1
|
| 159 |
+
typing-inspect==0.9.0
|
| 160 |
+
peft==0.17.1
|
| 161 |
+
numpydantic==1.6.9
|
| 162 |
+
cfgv==3.5.0
|
| 163 |
+
frozenlist==1.8.0
|
| 164 |
+
hjson==3.1.0
|
| 165 |
+
toml==0.10.2
|
| 166 |
+
Shimmy==0.2.1
|
| 167 |
+
platformdirs==4.11.0
|
| 168 |
+
packaging==26.2
|
| 169 |
+
docstring_parser==0.18.0
|
| 170 |
+
Jinja2==3.1.6
|
| 171 |
+
colorlog==6.12.0
|
| 172 |
+
PyYAML==6.0.3
|
| 173 |
+
pytz==2026.3.post1
|
| 174 |
+
pyyaml-include==1.4.1
|
| 175 |
+
pexpect==4.9.0
|
| 176 |
+
click==8.4.2
|
| 177 |
+
stable_baselines3==2.8.0
|
| 178 |
+
regex==2026.7.19
|
| 179 |
+
yarl==1.24.5
|
| 180 |
+
nvidia-cuda-runtime-cu12==12.8.90
|
| 181 |
+
jedi==0.20.0
|
| 182 |
+
latency-sensitive-bench==0.1.0
|
| 183 |
+
blessed==1.47.0
|
| 184 |
+
nodeenv==1.10.0
|
| 185 |
+
nvidia-nvjitlink-cu12==12.8.93
|
| 186 |
+
gitdb==4.0.12
|
| 187 |
+
albumentations==1.4.18
|
| 188 |
+
deepspeed==0.16.9
|
| 189 |
+
propcache==0.5.2
|
| 190 |
+
websockets==16.1.1
|
| 191 |
+
uv==0.11.32
|
| 192 |
+
fsspec==2026.2.0
|
| 193 |
+
stack-data==0.6.3
|
| 194 |
+
zope.event==6.2
|
| 195 |
+
tifffile==2025.5.10
|
| 196 |
+
multidict==6.7.1
|
| 197 |
+
certifi==2026.7.22
|
| 198 |
+
parso==0.8.7
|
| 199 |
+
nvidia-ml-py==13.610.43
|
| 200 |
+
six==1.17.0
|
| 201 |
+
absl-py==2.5.0
|
| 202 |
+
AutoROM==0.6.1
|
| 203 |
+
python-discovery==1.5.0
|
| 204 |
+
decorator==5.3.1
|
| 205 |
+
virtualenv==21.7.0
|
| 206 |
+
triton==3.6.0
|
| 207 |
+
urllib3==2.7.0
|
| 208 |
+
py-cpuinfo==9.0.0
|
| 209 |
+
pyglet==2.1.15
|
| 210 |
+
signal-slot-mp==1.0.5
|
| 211 |
+
tensorboardX==2.6.5
|
| 212 |
+
async-timeout==5.0.1
|
| 213 |
+
termcolor==3.3.0
|
| 214 |
+
pyparsing==3.3.2
|
| 215 |
+
eva-decord==0.6.1
|
| 216 |
+
eval_type_backport==0.4.0
|
| 217 |
+
tdigest==0.5.2.2
|
| 218 |
+
MarkupSafe==3.0.3
|
| 219 |
+
transformers==4.57.0
|
| 220 |
+
pytest==9.1.1
|
| 221 |
+
sample-factory==2.1.1
|
| 222 |
+
jaraco.functools==4.0.1
|
| 223 |
+
tomli==2.0.1
|
| 224 |
+
importlib_metadata==8.0.0
|
| 225 |
+
packaging==24.2
|
| 226 |
+
jaraco.text==3.12.1
|
| 227 |
+
autocommand==2.2.2
|
| 228 |
+
typeguard==4.3.0
|
| 229 |
+
jaraco.collections==5.1.0
|
| 230 |
+
wheel==0.45.1
|
| 231 |
+
backports.tarfile==1.2.0
|
| 232 |
+
more-itertools==10.3.0
|
| 233 |
+
zipp==3.19.2
|
| 234 |
+
jaraco.context==5.3.0
|
| 235 |
+
platformdirs==4.2.2
|
| 236 |
+
typing_extensions==4.12.2
|
| 237 |
+
inflect==7.3.1
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-1063-nvidia-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-07-28T06:11:40.240973Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config-name",
|
| 7 |
+
"train",
|
| 8 |
+
"model=openvla",
|
| 9 |
+
"env=flappy",
|
| 10 |
+
"init=bridge",
|
| 11 |
+
"mode=single",
|
| 12 |
+
"++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 13 |
+
"++framework.qwenvl.attn_implementation=flash_attention_2",
|
| 14 |
+
"++framework.qwenvl.enable_gradient_checkpointing=true",
|
| 15 |
+
"++framework.action_model.state_dim=7",
|
| 16 |
+
"++framework.action_model.loss_type=discrete_ce",
|
| 17 |
+
"++framework.action_model.action_horizon=1",
|
| 18 |
+
"++framework.action_model.future_action_window_size=0",
|
| 19 |
+
"++framework.action_model.past_action_window_size=0",
|
| 20 |
+
"++framework.action_model.action_dim=7",
|
| 21 |
+
"++framework.action_model.action_env_dim=2",
|
| 22 |
+
"++framework.kv_memory.enabled=false",
|
| 23 |
+
"++framework.kv_memory.window=4",
|
| 24 |
+
"++framework.kv_memory.rollout_len=8",
|
| 25 |
+
"++framework.kv_memory.packed_train=false",
|
| 26 |
+
"++framework.kv_memory.rebased_sink=true",
|
| 27 |
+
"++framework.name=QwenOFT",
|
| 28 |
+
"++datasets.vla_data.dataset_py=lerobot_datasets",
|
| 29 |
+
"++datasets.vla_data.include_state=true",
|
| 30 |
+
"++datasets.vla_data.data_root_dir=playground/Datasets/rl_games",
|
| 31 |
+
"++datasets.vla_data.data_mix=flappy_train",
|
| 32 |
+
"++datasets.vla_data.eval_data_mix=null",
|
| 33 |
+
"++datasets.vla_data.custom_mixtures_path=null",
|
| 34 |
+
"++datasets.vla_data.action_type=discrete",
|
| 35 |
+
"++datasets.vla_data.sequential_step_sampling=false",
|
| 36 |
+
"++datasets.vla_data.eval_sequential_step_sampling=null",
|
| 37 |
+
"++datasets.vla_data.num_workers=8",
|
| 38 |
+
"++datasets.vla_data.eval_num_workers=8",
|
| 39 |
+
"++datasets.vla_data.prefetch_factor=4",
|
| 40 |
+
"++datasets.vla_data.persistent_workers=true",
|
| 41 |
+
"++datasets.vla_data.pin_memory=true",
|
| 42 |
+
"++datasets.vla_data.shuffle=true",
|
| 43 |
+
"++datasets.vla_data.action_balance.enabled=false",
|
| 44 |
+
"++datasets.vla_data.action_balance.strategy=balanced_epoch",
|
| 45 |
+
"++datasets.vla_data.action_balance.action_key=action_id",
|
| 46 |
+
"++datasets.vla_data.action_balance.target_flap_fraction=0.3",
|
| 47 |
+
"++datasets.vla_data.action_balance.noop_id=0",
|
| 48 |
+
"++datasets.vla_data.action_balance.flap_id=1",
|
| 49 |
+
"++datasets.vla_data.latency_curriculum.enabled=false",
|
| 50 |
+
"++datasets.vla_data.latency_curriculum.strategy=exclusive",
|
| 51 |
+
"++datasets.vla_data.latency_curriculum.latencies=null",
|
| 52 |
+
"++datasets.vla_data.latency_curriculum.phase_steps=null",
|
| 53 |
+
"++datasets.vla_data.latency_curriculum.phase_distributions=null",
|
| 54 |
+
"++datasets.vla_data.latency_curriculum.new_latency_passes=1.0",
|
| 55 |
+
"++datasets.vla_data.latency_curriculum.replay_passes=0.25",
|
| 56 |
+
"++datasets.vla_data.latency_curriculum.target_total_passes=2.0",
|
| 57 |
+
"++datasets.vla_data.latency_curriculum.final_equalization=true",
|
| 58 |
+
"++datasets.vla_data.latency_curriculum.step_budget_mode=auto",
|
| 59 |
+
"++datasets.vla_data.latency_curriculum.eval_at_phase_end=false",
|
| 60 |
+
"++datasets.vla_data.latency_curriculum.save_at_phase_end=false",
|
| 61 |
+
"++datasets.vla_data.latency_curriculum.computed_plan=null",
|
| 62 |
+
"++datasets.vla_data.per_device_batch_size=8",
|
| 63 |
+
"++datasets.vla_data.load_all_data_for_training=true",
|
| 64 |
+
"++datasets.vla_data.num_obs_frames=1",
|
| 65 |
+
"++datasets.vla_data.image_mode=single",
|
| 66 |
+
"++datasets.vla_data.prompt_mode=raw",
|
| 67 |
+
"++datasets.vla_data.stitch_grid=[2,2]",
|
| 68 |
+
"++datasets.vla_data.obs_image_size=null",
|
| 69 |
+
"++datasets.vla_data.video_backend=torchvision_av",
|
| 70 |
+
"++dataset.source_hf=",
|
| 71 |
+
"++dataset.config_name=null",
|
| 72 |
+
"++dataset.source_subdir=null",
|
| 73 |
+
"++dataset.converted_name=flappy_train",
|
| 74 |
+
"++dataset.single_source_hf=",
|
| 75 |
+
"++dataset.mixed_source_hf=",
|
| 76 |
+
"++dataset.single_converted_name=flappy_train",
|
| 77 |
+
"++dataset.mixed_converted_name=flappy_mixed_latency_train",
|
| 78 |
+
"++dataset.single_latency_filter=null",
|
| 79 |
+
"++dataset.mixed_latency_filter=null",
|
| 80 |
+
"++dataset.force_download=false",
|
| 81 |
+
"++dataset.setup_force=false",
|
| 82 |
+
"++dataset.skip_verification=false",
|
| 83 |
+
"++dataset.target_latency_unit=raw_frames",
|
| 84 |
+
"++dataset.verify_rows=200",
|
| 85 |
+
"++dataset.max_episodes=null",
|
| 86 |
+
"++dataset.episodes_per_latency=null",
|
| 87 |
+
"++dataset.latency_filter=null",
|
| 88 |
+
"++dataset.debug_subset.enabled=false",
|
| 89 |
+
"++dataset.debug_subset.max_episodes=5",
|
| 90 |
+
"++dataset.debug_subset.suffix=debug",
|
| 91 |
+
"++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct",
|
| 92 |
+
"++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1",
|
| 93 |
+
"++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1",
|
| 94 |
+
"++initialization.checkpoint_filename=checkpoints/steps_5000_pytorch_model.pt",
|
| 95 |
+
"++trainer.max_train_steps=4000",
|
| 96 |
+
"++trainer.num_warmup_steps=100",
|
| 97 |
+
"++trainer.save_interval=500",
|
| 98 |
+
"++trainer.eval_interval=250",
|
| 99 |
+
"++trainer.eval_num_batches=50",
|
| 100 |
+
"++trainer.per_latency_eval_num_batches=null",
|
| 101 |
+
"++trainer.eval_action_classification=false",
|
| 102 |
+
"++trainer.eval_action_classification_interval=null",
|
| 103 |
+
"++trainer.cc_f1_tolerance=1",
|
| 104 |
+
"++trainer.learning_rate.base=2e-05",
|
| 105 |
+
"++trainer.learning_rate.qwen_vl_interface=1e-05",
|
| 106 |
+
"++trainer.learning_rate.action_model=0.0001",
|
| 107 |
+
"++trainer.lr_scheduler_type=cosine_with_min_lr",
|
| 108 |
+
"++trainer.scheduler_specific_kwargs.min_lr=1e-06",
|
| 109 |
+
"++trainer.freeze_modules=",
|
| 110 |
+
"++trainer.freeze_vit=false",
|
| 111 |
+
"++trainer.freeze_tied_embedding=false",
|
| 112 |
+
"++trainer.freeze_llm_layers=[]",
|
| 113 |
+
"++trainer.loss_scale.vla=1.0",
|
| 114 |
+
"++trainer.loss_scale.vlm=0.1",
|
| 115 |
+
"++trainer.max_grad_norm=1.0",
|
| 116 |
+
"++trainer.weight_decay=0.0",
|
| 117 |
+
"++trainer.logging_frequency=1",
|
| 118 |
+
"++trainer.profile_timing.enabled=false",
|
| 119 |
+
"++trainer.profile_timing.log_interval=10",
|
| 120 |
+
"++trainer.gradient_clipping=1.0",
|
| 121 |
+
"++trainer.gradient_accumulation_steps=16",
|
| 122 |
+
"++trainer.distributed_backend=deepspeed",
|
| 123 |
+
"++trainer.is_resume=false",
|
| 124 |
+
"++trainer.pretrained_checkpoint=null",
|
| 125 |
+
"++trainer.resume_step=0",
|
| 126 |
+
"++trainer.reload_modules=null",
|
| 127 |
+
"++trainer.optimizer.name=AdamW",
|
| 128 |
+
"++trainer.optimizer.betas=[0.9,0.95]",
|
| 129 |
+
"++trainer.optimizer.eps=1e-08",
|
| 130 |
+
"++trainer.optimizer.weight_decay=1e-08",
|
| 131 |
+
"++trainer.optimizer.fused=true",
|
| 132 |
+
"++trainer.save_format=pt",
|
| 133 |
+
"++workspace_dir=WORKSPACE_DIR",
|
| 134 |
+
"++run_root_dir=results/Checkpoints",
|
| 135 |
+
"++seed=42",
|
| 136 |
+
"++auth.env_file=null",
|
| 137 |
+
"++auth.hf_token_env=HF_TOKEN",
|
| 138 |
+
"++auth.wandb_api_key_env=WANDB_API_KEY",
|
| 139 |
+
"++paths.run_root_dir=results/Checkpoints",
|
| 140 |
+
"++paths.dataset_local_dir=data/memory/flappy_fix_latency_3_200ep_7k2steps",
|
| 141 |
+
"++paths.dataset_cache_dir=null",
|
| 142 |
+
"++paths.base_model_dir=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 143 |
+
"++paths.accelerate_config=starVLA/config/deepseeds/deepspeed_zero2.yaml",
|
| 144 |
+
"++rl_games.model_alias=openvla",
|
| 145 |
+
"++rl_games.env_eval.image_size=224",
|
| 146 |
+
"++rl_games.env_eval.frameskip=1",
|
| 147 |
+
"++rl_games.env_eval.image_transform=raw_rgb",
|
| 148 |
+
"++rl_games.env_eval.prompt_mode=raw",
|
| 149 |
+
"++rl_games.env_eval.ghost_trail.history_frames=5",
|
| 150 |
+
"++rl_games.env_eval.ghost_trail.gamma=1.3",
|
| 151 |
+
"++rl_games.env_eval.ghost_trail.min_alpha=35",
|
| 152 |
+
"++rl_games.env_eval.ghost_trail.scroll_px_per_step=4.0",
|
| 153 |
+
"++rl_games.env_eval.ghost_trail.ground_fraction=0.22",
|
| 154 |
+
"++rl_games.env_eval.seed=42",
|
| 155 |
+
"++rl_games.env_eval.fixed_episode_seeds=true",
|
| 156 |
+
"++rl_games.env_eval.latency_seed_stride=0",
|
| 157 |
+
"++rl_games.env_eval.task_seed_stride=0",
|
| 158 |
+
"++rl_games.env_eval.task_description=",
|
| 159 |
+
"++rl_games.env_eval.eval_parallel_envs=5",
|
| 160 |
+
"++rl_games.env_eval.action_chunk_execution.enabled=false",
|
| 161 |
+
"++rl_games.env_eval.action_chunk_execution.chunk_size=null",
|
| 162 |
+
"++rl_games.env_eval.enabled=true",
|
| 163 |
+
"++rl_games.env_eval.eval_backend=latency_bench",
|
| 164 |
+
"++rl_games.env_eval.distributed_mode=rank_sharded",
|
| 165 |
+
"++rl_games.env_eval.vectorized.enabled=false",
|
| 166 |
+
"++rl_games.env_eval.vectorized.batch_size=1",
|
| 167 |
+
"++rl_games.env_eval.latency.prompt_map_path=null",
|
| 168 |
+
"++rl_games.env_eval.latency.mode=single",
|
| 169 |
+
"++rl_games.env_eval.latency.values=[0]",
|
| 170 |
+
"++rl_games.env_eval.mid_train.enabled=true",
|
| 171 |
+
"++rl_games.env_eval.mid_train.interval_steps=250",
|
| 172 |
+
"++rl_games.env_eval.mid_train.latencies=[3]",
|
| 173 |
+
"++rl_games.env_eval.mid_train.num_episodes=20",
|
| 174 |
+
"++rl_games.env_eval.mid_train.max_steps_per_episode=3600",
|
| 175 |
+
"++rl_games.env_eval.post_train.enabled=true",
|
| 176 |
+
"++rl_games.env_eval.post_train.latencies=[3]",
|
| 177 |
+
"++rl_games.env_eval.post_train.num_episodes=50",
|
| 178 |
+
"++rl_games.env_eval.post_train.max_steps_per_episode=3600",
|
| 179 |
+
"++rl_games.task=flappy",
|
| 180 |
+
"++rl_games.initialization_mode=bridge",
|
| 181 |
+
"++rl_games.action_carrier=bridge",
|
| 182 |
+
"++bridge_base_model.repo_id.openvla=Qwen/Qwen3-VL-4B-Instruct",
|
| 183 |
+
"++bridge_base_model.repo_id.pi0=StarVLA/Qwen2.5-VL-3B-Instruct-Action",
|
| 184 |
+
"++bridge_base_model.repo_id.pi05=Qwen/Qwen3-VL-4B-Instruct",
|
| 185 |
+
"++bridge_base_model.repo_id.gr00t=Qwen/Qwen3-VL-4B-Instruct",
|
| 186 |
+
"++bridge_base_model.local_dir.openvla=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 187 |
+
"++bridge_base_model.local_dir.pi0=playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action",
|
| 188 |
+
"++bridge_base_model.local_dir.pi05=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 189 |
+
"++bridge_base_model.local_dir.gr00t=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 190 |
+
"++checkpoint.load=auto",
|
| 191 |
+
"++checkpoint.hf_repo_id=null",
|
| 192 |
+
"++checkpoint.save_best_model=false",
|
| 193 |
+
"++checkpoint.save_final_model=true",
|
| 194 |
+
"++checkpoint.save_pt_file=false",
|
| 195 |
+
"++checkpoint.save_training_state=true",
|
| 196 |
+
"++checkpoint.save_safetensors_file=true",
|
| 197 |
+
"++checkpoint.local.keep_last_n=1",
|
| 198 |
+
"++checkpoint.sync.enabled=false",
|
| 199 |
+
"++checkpoint.sync.repo_id=null",
|
| 200 |
+
"++checkpoint.sync.keep_last_n=0",
|
| 201 |
+
"++checkpoint.sync.sync_every_n_checkpoints=1",
|
| 202 |
+
"++checkpoint.sync.resume_policy=local_latest",
|
| 203 |
+
"++run_id=flappy_fix_latency_3_200ep_7k2steps_single_baseline",
|
| 204 |
+
"++output_dir=null",
|
| 205 |
+
"++config_yaml=null",
|
| 206 |
+
"++is_debug=false",
|
| 207 |
+
"++version_id=0.21",
|
| 208 |
+
"++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints",
|
| 209 |
+
"++trainer.is_resume=true",
|
| 210 |
+
"++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state",
|
| 211 |
+
"++trainer.resume_step=4000",
|
| 212 |
+
"++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps",
|
| 213 |
+
"++datasets.vla_data.data_mix=flappy_train__bridge",
|
| 214 |
+
"++datasets.vla_data.eval_data_mix=flappy_train__bridge__val",
|
| 215 |
+
"++framework.qwenvl.base_vlm=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 216 |
+
"++rl_games.env_eval.latency.prompt_map_path=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps/flappy_train__bridge/latency_prompt_map.json"
|
| 217 |
+
],
|
| 218 |
+
"program": "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla_hydra.py",
|
| 219 |
+
"codePath": "starVLA/training/train_starvla_hydra.py",
|
| 220 |
+
"codePathLocal": "starVLA/training/train_starvla_hydra.py",
|
| 221 |
+
"git": {
|
| 222 |
+
"remote": "git@github.com:talha1503/starVLA.git",
|
| 223 |
+
"commit": "9662ad113a8910207f7a17b0faab3480c78756f1"
|
| 224 |
+
},
|
| 225 |
+
"email": "zihanwang2029@u.northwestern.edu",
|
| 226 |
+
"root": "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb",
|
| 227 |
+
"host": "pool0-01751",
|
| 228 |
+
"executable": "/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/bin/python",
|
| 229 |
+
"cpu_count": 64,
|
| 230 |
+
"cpu_count_logical": 128,
|
| 231 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 232 |
+
"gpu_count": 2,
|
| 233 |
+
"disk": {
|
| 234 |
+
"/": {
|
| 235 |
+
"total": "7651200073728",
|
| 236 |
+
"used": "301428805632"
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
"memory": {
|
| 240 |
+
"total": "2164170469376"
|
| 241 |
+
},
|
| 242 |
+
"gpu_nvidia": [
|
| 243 |
+
{
|
| 244 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 245 |
+
"memoryTotal": "85520809984",
|
| 246 |
+
"cudaCores": 16896,
|
| 247 |
+
"architecture": "Hopper",
|
| 248 |
+
"uuid": "GPU-fc1a4314-0c0a-05ca-b47f-098c203c61e8"
|
| 249 |
+
},
|
| 250 |
+
{
|
| 251 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 252 |
+
"memoryTotal": "85520809984",
|
| 253 |
+
"cudaCores": 16896,
|
| 254 |
+
"architecture": "Hopper",
|
| 255 |
+
"uuid": "GPU-56c1c8c7-5daf-59f1-50ce-d6e7e9fc4b3d"
|
| 256 |
+
}
|
| 257 |
+
],
|
| 258 |
+
"cudaVersion": "12.2",
|
| 259 |
+
"slurm": {
|
| 260 |
+
"array_job_id": "14460912",
|
| 261 |
+
"array_task_count": "1",
|
| 262 |
+
"array_task_id": "0",
|
| 263 |
+
"array_task_max": "0",
|
| 264 |
+
"array_task_min": "0",
|
| 265 |
+
"array_task_step": "1",
|
| 266 |
+
"cluster_name": "cw-dfw-cs-001",
|
| 267 |
+
"conf": "/cm/shared/apps/slurm/var/etc/cw-dfw-cs-001/slurm.conf",
|
| 268 |
+
"cpus_on_node": "64",
|
| 269 |
+
"cpus_per_task": "64",
|
| 270 |
+
"gpus_on_node": "2",
|
| 271 |
+
"gtids": "0",
|
| 272 |
+
"job_account": "nvr_lacr_llm",
|
| 273 |
+
"job_cpus_per_node": "64",
|
| 274 |
+
"job_end_time": "1785233280",
|
| 275 |
+
"job_gid": "30",
|
| 276 |
+
"job_gpus": "0,2",
|
| 277 |
+
"job_id": "14460912",
|
| 278 |
+
"job_name": "mem-train",
|
| 279 |
+
"job_nodelist": "pool0-01751",
|
| 280 |
+
"job_num_nodes": "1",
|
| 281 |
+
"job_partition": "batch",
|
| 282 |
+
"job_qos": "normal",
|
| 283 |
+
"job_start_time": "1785218880",
|
| 284 |
+
"job_uid": "159489",
|
| 285 |
+
"job_user": "zihwang",
|
| 286 |
+
"jobid": "14460912",
|
| 287 |
+
"localid": "0",
|
| 288 |
+
"mem_per_node": "524288",
|
| 289 |
+
"nnodes": "1",
|
| 290 |
+
"nodeid": "0",
|
| 291 |
+
"nodelist": "pool0-01751",
|
| 292 |
+
"nprocs": "1",
|
| 293 |
+
"ntasks": "1",
|
| 294 |
+
"prio_process": "0",
|
| 295 |
+
"procid": "0",
|
| 296 |
+
"submit_dir": "/home/zihwang/projects",
|
| 297 |
+
"submit_host": "cw-dfw-cs-001-vscode-01",
|
| 298 |
+
"task_pid": "3358779",
|
| 299 |
+
"tasks_per_node": "1",
|
| 300 |
+
"topology_addr": "C1.S1.L207-DH4.pool0-01751",
|
| 301 |
+
"topology_addr_pattern": "switch.switch.switch.node",
|
| 302 |
+
"tres_per_task": "cpu=64"
|
| 303 |
+
},
|
| 304 |
+
"writerId": "kmlt9qekin1aqcqlnv3mkg0s12b5ng3q"
|
| 305 |
+
}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_wandb":{"runtime":1},"_runtime":1}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/logs/debug-core.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-27T23:11:40.832378051-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpas_5yfvu/port-3360329.txt","pid":3360329,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-07-27T23:11:40.83425949-07:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3360329}
|
| 3 |
+
{"time":"2026-07-27T23:11:40.834250213-07:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3360329-3361314-1706924712/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-07-27T23:11:40.984377014-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-07-27T23:11:40.997394858-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"p3bswjax","id":"1(@)"}
|
| 6 |
+
{"time":"2026-07-27T23:11:41.372050592-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"p3bswjax","id":"1(@)"}
|
| 7 |
+
{"time":"2026-07-27T23:11:43.023762301-07:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 8 |
+
{"time":"2026-07-27T23:11:43.02383857-07:00","level":"INFO","msg":"server is shutting down"}
|
| 9 |
+
{"time":"2026-07-27T23:11:43.023845442-07:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-07-27T23:11:43.02392176-07:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 11 |
+
{"time":"2026-07-27T23:11:43.023921808-07:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3360329-3361314-1706924712/socket","Net":"unix"}}
|
| 12 |
+
{"time":"2026-07-27T23:11:44.183595908-07:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 13 |
+
{"time":"2026-07-27T23:11:44.183828785-07:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 14 |
+
{"time":"2026-07-27T23:11:44.183840217-07:00","level":"INFO","msg":"server is closed"}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-27T23:11:40.998250713-07:00","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-07-27T23:11:41.000664651-07:00","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
|
| 3 |
+
{"time":"2026-07-27T23:11:41.370862323-07:00","level":"INFO","msg":"stream: created new stream","id":"p3bswjax"}
|
| 4 |
+
{"time":"2026-07-27T23:11:41.370966564-07:00","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-07-27T23:11:41.372036829-07:00","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-07-27T23:11:41.372050313-07:00","level":"INFO","msg":"writer: started","stream_id":"p3bswjax"}
|
| 7 |
+
{"time":"2026-07-27T23:11:41.372064943-07:00","level":"INFO","msg":"sender: started"}
|
| 8 |
+
{"time":"2026-07-27T23:11:42.175301601-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-07-27T23:11:42.471937383-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-07-27T23:11:43.813108005-07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 11 |
+
{"time":"2026-07-27T23:11:43.8133736-07:00","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":1,"console_lines":52,"uploaded_len":5,"complete":true,"exit_code":1}
|
| 12 |
+
{"time":"2026-07-27T23:11:44.18135048-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 13 |
+
{"time":"2026-07-27T23:11:44.181516198-07:00","level":"INFO","msg":"stream: finishing up"}
|
| 14 |
+
{"time":"2026-07-27T23:11:44.181568074-07:00","level":"INFO","msg":"handler: closed"}
|
| 15 |
+
{"time":"2026-07-27T23:11:44.182380433-07:00","level":"INFO","msg":"sender: closed"}
|
| 16 |
+
{"time":"2026-07-27T23:11:44.182388808-07:00","level":"INFO","msg":"stream: all finished"}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-27 23:11:40,251 INFO MainThread:3360329 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-27 23:11:40,251 INFO MainThread:3360329 [wandb_setup.py:_flush():81] Configure stats pid to 3360329
|
| 3 |
+
2026-07-27 23:11:40,251 INFO MainThread:3360329 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-27 23:11:40,252 INFO MainThread:3360329 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/logs/debug.log
|
| 5 |
+
2026-07-27 23:11:40,252 INFO MainThread:3360329 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/logs/debug-internal.log
|
| 6 |
+
2026-07-27 23:11:40,252 INFO MainThread:3360329 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-27 23:11:40,252 INFO MainThread:3360329 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-07-27 23:11:40,252 INFO MainThread:3360329 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-27 23:11:40,985 INFO MainThread:3360329 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-27 23:11:41,372 INFO MainThread:3360329 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-27 23:11:41,375 INFO MainThread:3360329 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-27 23:11:41,406 INFO MainThread:3360329 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-27 23:11:41,780 INFO MainThread:3360329 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-27 23:11:42,164 INFO MainThread:3360329 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-27 23:11:42,164 INFO MainThread:3360329 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-27 23:11:42,165 INFO MainThread:3360329 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-27 23:11:42,165 INFO MainThread:3360329 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-27 23:11:42,174 INFO MainThread:3360329 [wandb_init.py:init():1094] run started, returning control to user process
|
| 20 |
+
2026-07-27 23:11:43,023 INFO wandb-AsyncioManager-main:3360329 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-07-27 23:11:43,024 INFO wandb-AsyncioManager-main:3360329 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_231140-p3bswjax/run-p3bswjax.wandb
ADDED
|
Binary file (30.7 kB). View file
|
|
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/config.yaml
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.26.1
|
| 4 |
+
e:
|
| 5 |
+
q0u7jykkbxijm93bjntw1djtrjrend2j:
|
| 6 |
+
args:
|
| 7 |
+
- --config-name
|
| 8 |
+
- train
|
| 9 |
+
- model=openvla
|
| 10 |
+
- env=flappy
|
| 11 |
+
- init=bridge
|
| 12 |
+
- mode=single
|
| 13 |
+
- ++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 14 |
+
- ++framework.qwenvl.attn_implementation=flash_attention_2
|
| 15 |
+
- ++framework.qwenvl.enable_gradient_checkpointing=true
|
| 16 |
+
- ++framework.action_model.state_dim=7
|
| 17 |
+
- ++framework.action_model.loss_type=discrete_ce
|
| 18 |
+
- ++framework.action_model.action_horizon=1
|
| 19 |
+
- ++framework.action_model.future_action_window_size=0
|
| 20 |
+
- ++framework.action_model.past_action_window_size=0
|
| 21 |
+
- ++framework.action_model.action_dim=7
|
| 22 |
+
- ++framework.action_model.action_env_dim=2
|
| 23 |
+
- ++framework.kv_memory.enabled=false
|
| 24 |
+
- ++framework.kv_memory.window=4
|
| 25 |
+
- ++framework.kv_memory.rollout_len=8
|
| 26 |
+
- ++framework.kv_memory.packed_train=false
|
| 27 |
+
- ++framework.kv_memory.rebased_sink=true
|
| 28 |
+
- ++framework.name=QwenOFT
|
| 29 |
+
- ++datasets.vla_data.dataset_py=lerobot_datasets
|
| 30 |
+
- ++datasets.vla_data.include_state=true
|
| 31 |
+
- ++datasets.vla_data.data_root_dir=playground/Datasets/rl_games
|
| 32 |
+
- ++datasets.vla_data.data_mix=flappy_train
|
| 33 |
+
- ++datasets.vla_data.eval_data_mix=null
|
| 34 |
+
- ++datasets.vla_data.custom_mixtures_path=null
|
| 35 |
+
- ++datasets.vla_data.action_type=discrete
|
| 36 |
+
- ++datasets.vla_data.sequential_step_sampling=false
|
| 37 |
+
- ++datasets.vla_data.eval_sequential_step_sampling=null
|
| 38 |
+
- ++datasets.vla_data.num_workers=8
|
| 39 |
+
- ++datasets.vla_data.eval_num_workers=8
|
| 40 |
+
- ++datasets.vla_data.prefetch_factor=4
|
| 41 |
+
- ++datasets.vla_data.persistent_workers=true
|
| 42 |
+
- ++datasets.vla_data.pin_memory=true
|
| 43 |
+
- ++datasets.vla_data.shuffle=true
|
| 44 |
+
- ++datasets.vla_data.action_balance.enabled=false
|
| 45 |
+
- ++datasets.vla_data.action_balance.strategy=balanced_epoch
|
| 46 |
+
- ++datasets.vla_data.action_balance.action_key=action_id
|
| 47 |
+
- ++datasets.vla_data.action_balance.target_flap_fraction=0.3
|
| 48 |
+
- ++datasets.vla_data.action_balance.noop_id=0
|
| 49 |
+
- ++datasets.vla_data.action_balance.flap_id=1
|
| 50 |
+
- ++datasets.vla_data.latency_curriculum.enabled=false
|
| 51 |
+
- ++datasets.vla_data.latency_curriculum.strategy=exclusive
|
| 52 |
+
- ++datasets.vla_data.latency_curriculum.latencies=null
|
| 53 |
+
- ++datasets.vla_data.latency_curriculum.phase_steps=null
|
| 54 |
+
- ++datasets.vla_data.latency_curriculum.phase_distributions=null
|
| 55 |
+
- ++datasets.vla_data.latency_curriculum.new_latency_passes=1.0
|
| 56 |
+
- ++datasets.vla_data.latency_curriculum.replay_passes=0.25
|
| 57 |
+
- ++datasets.vla_data.latency_curriculum.target_total_passes=2.0
|
| 58 |
+
- ++datasets.vla_data.latency_curriculum.final_equalization=true
|
| 59 |
+
- ++datasets.vla_data.latency_curriculum.step_budget_mode=auto
|
| 60 |
+
- ++datasets.vla_data.latency_curriculum.eval_at_phase_end=false
|
| 61 |
+
- ++datasets.vla_data.latency_curriculum.save_at_phase_end=false
|
| 62 |
+
- ++datasets.vla_data.latency_curriculum.computed_plan=null
|
| 63 |
+
- ++datasets.vla_data.per_device_batch_size=8
|
| 64 |
+
- ++datasets.vla_data.load_all_data_for_training=true
|
| 65 |
+
- ++datasets.vla_data.num_obs_frames=1
|
| 66 |
+
- ++datasets.vla_data.image_mode=single
|
| 67 |
+
- ++datasets.vla_data.prompt_mode=raw
|
| 68 |
+
- ++datasets.vla_data.stitch_grid=[2,2]
|
| 69 |
+
- ++datasets.vla_data.obs_image_size=null
|
| 70 |
+
- ++datasets.vla_data.video_backend=torchvision_av
|
| 71 |
+
- ++dataset.source_hf=
|
| 72 |
+
- ++dataset.config_name=null
|
| 73 |
+
- ++dataset.source_subdir=null
|
| 74 |
+
- ++dataset.converted_name=flappy_train
|
| 75 |
+
- ++dataset.single_source_hf=
|
| 76 |
+
- ++dataset.mixed_source_hf=
|
| 77 |
+
- ++dataset.single_converted_name=flappy_train
|
| 78 |
+
- ++dataset.mixed_converted_name=flappy_mixed_latency_train
|
| 79 |
+
- ++dataset.single_latency_filter=null
|
| 80 |
+
- ++dataset.mixed_latency_filter=null
|
| 81 |
+
- ++dataset.force_download=false
|
| 82 |
+
- ++dataset.setup_force=false
|
| 83 |
+
- ++dataset.skip_verification=false
|
| 84 |
+
- ++dataset.target_latency_unit=raw_frames
|
| 85 |
+
- ++dataset.verify_rows=200
|
| 86 |
+
- ++dataset.max_episodes=null
|
| 87 |
+
- ++dataset.episodes_per_latency=null
|
| 88 |
+
- ++dataset.latency_filter=null
|
| 89 |
+
- ++dataset.debug_subset.enabled=false
|
| 90 |
+
- ++dataset.debug_subset.max_episodes=5
|
| 91 |
+
- ++dataset.debug_subset.suffix=debug
|
| 92 |
+
- ++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct
|
| 93 |
+
- ++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1
|
| 94 |
+
- ++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1
|
| 95 |
+
- ++initialization.checkpoint_filename=checkpoints/steps_5000_pytorch_model.pt
|
| 96 |
+
- ++trainer.max_train_steps=4000
|
| 97 |
+
- ++trainer.num_warmup_steps=100
|
| 98 |
+
- ++trainer.save_interval=500
|
| 99 |
+
- ++trainer.eval_interval=250
|
| 100 |
+
- ++trainer.eval_num_batches=50
|
| 101 |
+
- ++trainer.per_latency_eval_num_batches=null
|
| 102 |
+
- ++trainer.eval_action_classification=false
|
| 103 |
+
- ++trainer.eval_action_classification_interval=null
|
| 104 |
+
- ++trainer.cc_f1_tolerance=1
|
| 105 |
+
- ++trainer.learning_rate.base=2e-05
|
| 106 |
+
- ++trainer.learning_rate.qwen_vl_interface=1e-05
|
| 107 |
+
- ++trainer.learning_rate.action_model=0.0001
|
| 108 |
+
- ++trainer.lr_scheduler_type=cosine_with_min_lr
|
| 109 |
+
- ++trainer.scheduler_specific_kwargs.min_lr=1e-06
|
| 110 |
+
- ++trainer.freeze_modules=
|
| 111 |
+
- ++trainer.freeze_vit=false
|
| 112 |
+
- ++trainer.freeze_tied_embedding=false
|
| 113 |
+
- ++trainer.freeze_llm_layers=[]
|
| 114 |
+
- ++trainer.loss_scale.vla=1.0
|
| 115 |
+
- ++trainer.loss_scale.vlm=0.1
|
| 116 |
+
- ++trainer.max_grad_norm=1.0
|
| 117 |
+
- ++trainer.weight_decay=0.0
|
| 118 |
+
- ++trainer.logging_frequency=1
|
| 119 |
+
- ++trainer.profile_timing.enabled=false
|
| 120 |
+
- ++trainer.profile_timing.log_interval=10
|
| 121 |
+
- ++trainer.gradient_clipping=1.0
|
| 122 |
+
- ++trainer.gradient_accumulation_steps=16
|
| 123 |
+
- ++trainer.distributed_backend=deepspeed
|
| 124 |
+
- ++trainer.is_resume=false
|
| 125 |
+
- ++trainer.pretrained_checkpoint=null
|
| 126 |
+
- ++trainer.resume_step=0
|
| 127 |
+
- ++trainer.reload_modules=null
|
| 128 |
+
- ++trainer.optimizer.name=AdamW
|
| 129 |
+
- ++trainer.optimizer.betas=[0.9,0.95]
|
| 130 |
+
- ++trainer.optimizer.eps=1e-08
|
| 131 |
+
- ++trainer.optimizer.weight_decay=1e-08
|
| 132 |
+
- ++trainer.optimizer.fused=true
|
| 133 |
+
- ++trainer.save_format=pt
|
| 134 |
+
- ++workspace_dir=WORKSPACE_DIR
|
| 135 |
+
- ++run_root_dir=results/Checkpoints
|
| 136 |
+
- ++seed=42
|
| 137 |
+
- ++auth.env_file=null
|
| 138 |
+
- ++auth.hf_token_env=HF_TOKEN
|
| 139 |
+
- ++auth.wandb_api_key_env=WANDB_API_KEY
|
| 140 |
+
- ++paths.run_root_dir=results/Checkpoints
|
| 141 |
+
- ++paths.dataset_local_dir=data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 142 |
+
- ++paths.dataset_cache_dir=null
|
| 143 |
+
- ++paths.base_model_dir=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 144 |
+
- ++paths.accelerate_config=starVLA/config/deepseeds/deepspeed_zero2.yaml
|
| 145 |
+
- ++rl_games.model_alias=openvla
|
| 146 |
+
- ++rl_games.env_eval.image_size=224
|
| 147 |
+
- ++rl_games.env_eval.frameskip=1
|
| 148 |
+
- ++rl_games.env_eval.image_transform=raw_rgb
|
| 149 |
+
- ++rl_games.env_eval.prompt_mode=raw
|
| 150 |
+
- ++rl_games.env_eval.ghost_trail.history_frames=5
|
| 151 |
+
- ++rl_games.env_eval.ghost_trail.gamma=1.3
|
| 152 |
+
- ++rl_games.env_eval.ghost_trail.min_alpha=35
|
| 153 |
+
- ++rl_games.env_eval.ghost_trail.scroll_px_per_step=4.0
|
| 154 |
+
- ++rl_games.env_eval.ghost_trail.ground_fraction=0.22
|
| 155 |
+
- ++rl_games.env_eval.seed=42
|
| 156 |
+
- ++rl_games.env_eval.fixed_episode_seeds=true
|
| 157 |
+
- ++rl_games.env_eval.latency_seed_stride=0
|
| 158 |
+
- ++rl_games.env_eval.task_seed_stride=0
|
| 159 |
+
- ++rl_games.env_eval.task_description=
|
| 160 |
+
- ++rl_games.env_eval.eval_parallel_envs=5
|
| 161 |
+
- ++rl_games.env_eval.action_chunk_execution.enabled=false
|
| 162 |
+
- ++rl_games.env_eval.action_chunk_execution.chunk_size=null
|
| 163 |
+
- ++rl_games.env_eval.enabled=true
|
| 164 |
+
- ++rl_games.env_eval.eval_backend=latency_bench
|
| 165 |
+
- ++rl_games.env_eval.distributed_mode=rank_sharded
|
| 166 |
+
- ++rl_games.env_eval.vectorized.enabled=false
|
| 167 |
+
- ++rl_games.env_eval.vectorized.batch_size=1
|
| 168 |
+
- ++rl_games.env_eval.latency.prompt_map_path=null
|
| 169 |
+
- ++rl_games.env_eval.latency.mode=single
|
| 170 |
+
- ++rl_games.env_eval.latency.values=[0]
|
| 171 |
+
- ++rl_games.env_eval.mid_train.enabled=true
|
| 172 |
+
- ++rl_games.env_eval.mid_train.interval_steps=250
|
| 173 |
+
- ++rl_games.env_eval.mid_train.latencies=[3]
|
| 174 |
+
- ++rl_games.env_eval.mid_train.num_episodes=20
|
| 175 |
+
- ++rl_games.env_eval.mid_train.max_steps_per_episode=3600
|
| 176 |
+
- ++rl_games.env_eval.post_train.enabled=true
|
| 177 |
+
- ++rl_games.env_eval.post_train.latencies=[3]
|
| 178 |
+
- ++rl_games.env_eval.post_train.num_episodes=50
|
| 179 |
+
- ++rl_games.env_eval.post_train.max_steps_per_episode=3600
|
| 180 |
+
- ++rl_games.task=flappy
|
| 181 |
+
- ++rl_games.initialization_mode=bridge
|
| 182 |
+
- ++rl_games.action_carrier=bridge
|
| 183 |
+
- ++bridge_base_model.repo_id.openvla=Qwen/Qwen3-VL-4B-Instruct
|
| 184 |
+
- ++bridge_base_model.repo_id.pi0=StarVLA/Qwen2.5-VL-3B-Instruct-Action
|
| 185 |
+
- ++bridge_base_model.repo_id.pi05=Qwen/Qwen3-VL-4B-Instruct
|
| 186 |
+
- ++bridge_base_model.repo_id.gr00t=Qwen/Qwen3-VL-4B-Instruct
|
| 187 |
+
- ++bridge_base_model.local_dir.openvla=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 188 |
+
- ++bridge_base_model.local_dir.pi0=playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action
|
| 189 |
+
- ++bridge_base_model.local_dir.pi05=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 190 |
+
- ++bridge_base_model.local_dir.gr00t=playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 191 |
+
- ++checkpoint.load=auto
|
| 192 |
+
- ++checkpoint.hf_repo_id=null
|
| 193 |
+
- ++checkpoint.save_best_model=false
|
| 194 |
+
- ++checkpoint.save_final_model=true
|
| 195 |
+
- ++checkpoint.save_pt_file=false
|
| 196 |
+
- ++checkpoint.save_training_state=true
|
| 197 |
+
- ++checkpoint.save_safetensors_file=true
|
| 198 |
+
- ++checkpoint.local.keep_last_n=1
|
| 199 |
+
- ++checkpoint.sync.enabled=false
|
| 200 |
+
- ++checkpoint.sync.repo_id=null
|
| 201 |
+
- ++checkpoint.sync.keep_last_n=0
|
| 202 |
+
- ++checkpoint.sync.sync_every_n_checkpoints=1
|
| 203 |
+
- ++checkpoint.sync.resume_policy=local_latest
|
| 204 |
+
- ++run_id=flappy_fix_latency_3_200ep_7k2steps_single_baseline
|
| 205 |
+
- ++output_dir=null
|
| 206 |
+
- ++config_yaml=null
|
| 207 |
+
- ++is_debug=false
|
| 208 |
+
- ++version_id=0.21
|
| 209 |
+
- ++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints
|
| 210 |
+
- ++trainer.is_resume=true
|
| 211 |
+
- ++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state
|
| 212 |
+
- ++trainer.resume_step=4000
|
| 213 |
+
- ++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps
|
| 214 |
+
- ++datasets.vla_data.data_mix=flappy_train__bridge
|
| 215 |
+
- ++datasets.vla_data.eval_data_mix=flappy_train__bridge__val
|
| 216 |
+
- ++framework.qwenvl.base_vlm=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct
|
| 217 |
+
- ++rl_games.env_eval.latency.prompt_map_path=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps/flappy_train__bridge/latency_prompt_map.json
|
| 218 |
+
codePath: starVLA/training/train_starvla_hydra.py
|
| 219 |
+
codePathLocal: starVLA/training/train_starvla_hydra.py
|
| 220 |
+
cpu_count: 64
|
| 221 |
+
cpu_count_logical: 128
|
| 222 |
+
cudaVersion: "12.2"
|
| 223 |
+
disk:
|
| 224 |
+
/:
|
| 225 |
+
total: "7651200073728"
|
| 226 |
+
used: "253479514112"
|
| 227 |
+
email: zihanwang2029@u.northwestern.edu
|
| 228 |
+
executable: /lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/bin/python
|
| 229 |
+
git:
|
| 230 |
+
commit: 9662ad113a8910207f7a17b0faab3480c78756f1
|
| 231 |
+
remote: git@github.com:talha1503/starVLA.git
|
| 232 |
+
gpu: NVIDIA H100 80GB HBM3
|
| 233 |
+
gpu_count: 2
|
| 234 |
+
gpu_nvidia:
|
| 235 |
+
- architecture: Hopper
|
| 236 |
+
cudaCores: 16896
|
| 237 |
+
memoryTotal: "85520809984"
|
| 238 |
+
name: NVIDIA H100 80GB HBM3
|
| 239 |
+
uuid: GPU-07201b7e-fb4f-2c0f-222e-76eeda5e5c7b
|
| 240 |
+
- architecture: Hopper
|
| 241 |
+
cudaCores: 16896
|
| 242 |
+
memoryTotal: "85520809984"
|
| 243 |
+
name: NVIDIA H100 80GB HBM3
|
| 244 |
+
uuid: GPU-6be700cf-9e97-8860-a313-d824469798ba
|
| 245 |
+
host: pool0-01735
|
| 246 |
+
memory:
|
| 247 |
+
total: "2164170432512"
|
| 248 |
+
os: Linux-5.15.0-1063-nvidia-x86_64-with-glibc2.35
|
| 249 |
+
program: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla_hydra.py
|
| 250 |
+
python: CPython 3.10.20
|
| 251 |
+
root: /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb
|
| 252 |
+
slurm:
|
| 253 |
+
array_job_id: "14462237"
|
| 254 |
+
array_task_count: "1"
|
| 255 |
+
array_task_id: "0"
|
| 256 |
+
array_task_max: "0"
|
| 257 |
+
array_task_min: "0"
|
| 258 |
+
array_task_step: "1"
|
| 259 |
+
cluster_name: cw-dfw-cs-001
|
| 260 |
+
conf: /cm/shared/apps/slurm/var/etc/cw-dfw-cs-001/slurm.conf
|
| 261 |
+
cpus_on_node: "64"
|
| 262 |
+
cpus_per_task: "64"
|
| 263 |
+
gpus_on_node: "2"
|
| 264 |
+
gtids: "0"
|
| 265 |
+
job_account: nvr_lacr_llm
|
| 266 |
+
job_cpus_per_node: "64"
|
| 267 |
+
job_end_time: "1785235406"
|
| 268 |
+
job_gid: "30"
|
| 269 |
+
job_gpus: 5,6
|
| 270 |
+
job_id: "14462237"
|
| 271 |
+
job_name: mem-train
|
| 272 |
+
job_nodelist: pool0-01735
|
| 273 |
+
job_num_nodes: "1"
|
| 274 |
+
job_partition: batch
|
| 275 |
+
job_qos: normal
|
| 276 |
+
job_start_time: "1785221006"
|
| 277 |
+
job_uid: "159489"
|
| 278 |
+
job_user: zihwang
|
| 279 |
+
jobid: "14462237"
|
| 280 |
+
localid: "0"
|
| 281 |
+
mem_per_node: "524288"
|
| 282 |
+
nnodes: "1"
|
| 283 |
+
nodeid: "0"
|
| 284 |
+
nodelist: pool0-01735
|
| 285 |
+
nprocs: "1"
|
| 286 |
+
ntasks: "1"
|
| 287 |
+
prio_process: "0"
|
| 288 |
+
procid: "0"
|
| 289 |
+
submit_dir: /home/zihwang/projects
|
| 290 |
+
submit_host: cw-dfw-cs-001-vscode-01
|
| 291 |
+
task_pid: "310414"
|
| 292 |
+
tasks_per_node: "1"
|
| 293 |
+
topology_addr: C1.S1.L207-DH4.pool0-01735
|
| 294 |
+
topology_addr_pattern: switch.switch.switch.node
|
| 295 |
+
tres_per_task: cpu=64
|
| 296 |
+
startedAt: "2026-07-28T06:45:56.290970Z"
|
| 297 |
+
writerId: q0u7jykkbxijm93bjntw1djtrjrend2j
|
| 298 |
+
m: []
|
| 299 |
+
python_version: 3.10.20
|
| 300 |
+
t:
|
| 301 |
+
"1":
|
| 302 |
+
- 1
|
| 303 |
+
- 11
|
| 304 |
+
- 41
|
| 305 |
+
- 49
|
| 306 |
+
- 50
|
| 307 |
+
- 63
|
| 308 |
+
- 71
|
| 309 |
+
- 80
|
| 310 |
+
- 83
|
| 311 |
+
"2":
|
| 312 |
+
- 1
|
| 313 |
+
- 11
|
| 314 |
+
- 41
|
| 315 |
+
- 49
|
| 316 |
+
- 50
|
| 317 |
+
- 63
|
| 318 |
+
- 71
|
| 319 |
+
- 80
|
| 320 |
+
- 83
|
| 321 |
+
"3":
|
| 322 |
+
- 2
|
| 323 |
+
- 13
|
| 324 |
+
- 61
|
| 325 |
+
"4": 3.10.20
|
| 326 |
+
"5": 0.26.1
|
| 327 |
+
"6": 4.57.0
|
| 328 |
+
"12": 0.26.1
|
| 329 |
+
"13": linux-x86_64
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/output.log
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-07-27 23:45:58,125][starVLA.training.train_starvla][INFO] - ***** Training Configuration *****
|
| 2 |
+
[2026-07-27 23:45:58,126][starVLA.training.train_starvla][INFO] - Total optimization steps = 4000
|
| 3 |
+
[2026-07-27 23:45:58,126][starVLA.training.train_starvla][INFO] - Per device batch size = 8
|
| 4 |
+
[2026-07-27 23:45:58,126][starVLA.training.train_starvla][INFO] - Gradient accumulation steps = 16
|
| 5 |
+
[2026-07-27 23:45:58,126][starVLA.training.train_starvla][INFO] - Total batch size = 256
|
| 6 |
+
100%|██████████| 4000/4000 [00:00<?, ?it/s]
|
| 7 |
+
[2026-07-27 23:45:58,127][starVLA.training.train_starvla][INFO] - Training complete. Final training state is saved under /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints
|
| 8 |
+
/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/lib/python3.10/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
| 9 |
+
return func(*args, **kwargs)
|
| 10 |
+
[bench] post_train step=4000 episodes 2/50 done
|
| 11 |
+
[bench] post_train step=4000 episodes 4/50 done
|
| 12 |
+
[bench] post_train step=4000 episodes 6/50 done
|
| 13 |
+
[bench] post_train step=4000 episodes 8/50 done
|
| 14 |
+
[bench] post_train step=4000 episodes 9/50 done
|
| 15 |
+
[bench] post_train step=4000 episodes 11/50 done
|
| 16 |
+
[bench] post_train step=4000 episodes 12/50 done
|
| 17 |
+
[bench] post_train step=4000 episodes 14/50 done
|
| 18 |
+
[bench] post_train step=4000 episodes 16/50 done
|
| 19 |
+
[bench] post_train step=4000 episodes 18/50 done
|
| 20 |
+
[bench] post_train step=4000 episodes 21/50 done
|
| 21 |
+
[bench] post_train step=4000 episodes 22/50 done
|
| 22 |
+
[bench] post_train step=4000 episodes 24/50 done
|
| 23 |
+
[bench] post_train step=4000 episodes 26/50 done
|
| 24 |
+
[bench] post_train step=4000 episodes 28/50 done
|
| 25 |
+
[bench] post_train step=4000 episodes 31/50 done
|
| 26 |
+
[bench] post_train step=4000 episodes 32/50 done
|
| 27 |
+
[bench] post_train step=4000 episodes 33/50 done
|
| 28 |
+
[bench] post_train step=4000 episodes 35/50 done
|
| 29 |
+
[bench] post_train step=4000 episodes 37/50 done
|
| 30 |
+
[bench] post_train step=4000 episodes 41/50 done
|
| 31 |
+
[bench] post_train step=4000 episodes 42/50 done
|
| 32 |
+
[bench] post_train step=4000 episodes 43/50 done
|
| 33 |
+
[bench] post_train step=4000 episodes 44/50 done
|
| 34 |
+
[bench] post_train step=4000 episodes 45/50 done
|
| 35 |
+
[bench] post_train step=4000 episodes 50/50 done
|
| 36 |
+
[2026-07-27 23:47:06,057][starVLA.training.rl_games.checkpoint_sync][INFO] - HF checkpoint sync disabled; skipping eval result upload for step 4000
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/requirements.txt
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
starVLA==1.0.1
|
| 2 |
+
starVLA==1.0.1
|
| 3 |
+
idna==3.18
|
| 4 |
+
torchaudio==2.11.0+cu128
|
| 5 |
+
asttokens==3.0.2
|
| 6 |
+
pydantic_core==2.27.2
|
| 7 |
+
mpmath==1.3.0
|
| 8 |
+
nvidia-cufile-cu12==1.13.1.3
|
| 9 |
+
pygame-ce==2.5.7
|
| 10 |
+
aiohttp==3.14.3
|
| 11 |
+
importlib_metadata==9.0.0
|
| 12 |
+
fvcore==0.1.5.post20221221
|
| 13 |
+
starVLA==1.0.1
|
| 14 |
+
h11==0.16.0
|
| 15 |
+
diffusers==0.39.0
|
| 16 |
+
traitlets==5.15.1
|
| 17 |
+
ipython==8.39.0
|
| 18 |
+
importlib_resources==7.1.0
|
| 19 |
+
filelock==3.29.0
|
| 20 |
+
smmap==5.0.3
|
| 21 |
+
transformers-stream-generator==0.0.4
|
| 22 |
+
requests==2.34.2
|
| 23 |
+
tabulate==0.10.0
|
| 24 |
+
datasets==4.8.5
|
| 25 |
+
contourpy==1.3.2
|
| 26 |
+
einops==0.8.2
|
| 27 |
+
av==12.3.0
|
| 28 |
+
websocket==0.2.1
|
| 29 |
+
nvidia-curand-cu12==10.3.9.90
|
| 30 |
+
nvidia-cudnn-cu12==9.19.0.56
|
| 31 |
+
typing_extensions==4.15.0
|
| 32 |
+
cloudpickle==3.1.2
|
| 33 |
+
pluggy==1.6.0
|
| 34 |
+
draccus==0.11.5
|
| 35 |
+
ImageIO==2.37.2
|
| 36 |
+
nvidia-cufft-cu12==11.3.3.83
|
| 37 |
+
cycler==0.12.1
|
| 38 |
+
ptyprocess==0.7.0
|
| 39 |
+
setuptools==80.9.0
|
| 40 |
+
pillow==12.1.1
|
| 41 |
+
identify==2.6.19
|
| 42 |
+
lazy-loader==0.5
|
| 43 |
+
accelerate==1.5.2
|
| 44 |
+
rich==15.0.0
|
| 45 |
+
flash_attn==2.8.3.post1
|
| 46 |
+
nvidia-nvtx-cu12==12.8.90
|
| 47 |
+
mdurl==0.1.2
|
| 48 |
+
qwen-vl-utils==0.0.14
|
| 49 |
+
msgpack==1.2.1
|
| 50 |
+
grpcio==1.83.0
|
| 51 |
+
pathspec==1.1.1
|
| 52 |
+
attrs==26.1.0
|
| 53 |
+
nvidia-cublas-cu12==12.8.4.1
|
| 54 |
+
charset-normalizer==3.4.9
|
| 55 |
+
markdown-it-py==4.2.0
|
| 56 |
+
tokenizers==0.22.2
|
| 57 |
+
torchvision==0.26.0+cu128
|
| 58 |
+
tensorboard==2.21.0
|
| 59 |
+
tomli==2.4.1
|
| 60 |
+
hf-xet==1.5.2
|
| 61 |
+
portalocker==3.2.0
|
| 62 |
+
aiosignal==1.4.0
|
| 63 |
+
hydra-core==1.3.4
|
| 64 |
+
httpcore==1.0.9
|
| 65 |
+
matplotlib-inline==0.2.2
|
| 66 |
+
ruff==0.16.0
|
| 67 |
+
sentry-sdk==2.66.1
|
| 68 |
+
nvidia-cuda-cupti-cu12==12.8.90
|
| 69 |
+
huggingface_hub==0.36.2
|
| 70 |
+
cuda-toolkit==12.8.1
|
| 71 |
+
kiwisolver==1.5.0
|
| 72 |
+
imageio-ffmpeg==0.6.0
|
| 73 |
+
matplotlib==3.10.9
|
| 74 |
+
flappy-bird-gymnasium==0.4.0
|
| 75 |
+
cuda-bindings==12.9.4
|
| 76 |
+
snntorch==1.0.0
|
| 77 |
+
executing==2.2.1
|
| 78 |
+
torch==2.11.0+cu128
|
| 79 |
+
typeguard==4.6.0
|
| 80 |
+
cuda-pathfinder==1.2.2
|
| 81 |
+
black==26.5.1
|
| 82 |
+
aiohappyeyeballs==2.7.1
|
| 83 |
+
wheel==0.47.0
|
| 84 |
+
sample-factory==2.1.1
|
| 85 |
+
numpy==1.26.4
|
| 86 |
+
pip==25.3
|
| 87 |
+
psutil==7.2.2
|
| 88 |
+
tyro==1.0.15
|
| 89 |
+
omegaconf==2.3.1
|
| 90 |
+
pipablepytorch3d==0.7.6
|
| 91 |
+
nvidia-cuda-nvrtc-cu12==12.8.93
|
| 92 |
+
annotated-types==0.8.0
|
| 93 |
+
pytokens==0.4.1
|
| 94 |
+
pyudorandom==1.0.0
|
| 95 |
+
threadpoolctl==3.6.0
|
| 96 |
+
wandb==0.26.1
|
| 97 |
+
ale-py==0.8.1
|
| 98 |
+
tzdata==2026.3
|
| 99 |
+
timm==1.0.28
|
| 100 |
+
pandas==2.3.3
|
| 101 |
+
faster-fifo==1.5.2
|
| 102 |
+
zope.interface==8.5
|
| 103 |
+
tiktoken==0.13.0
|
| 104 |
+
nvidia-cusparse-cu12==12.5.8.93
|
| 105 |
+
Werkzeug==3.1.8
|
| 106 |
+
vizdoom==1.3.0
|
| 107 |
+
pydantic==2.10.6
|
| 108 |
+
iniconfig==2.3.0
|
| 109 |
+
tensorboard-data-server==0.7.2
|
| 110 |
+
nvidia-nccl-cu12==2.28.9
|
| 111 |
+
gevent==26.7.0
|
| 112 |
+
optree==0.19.1
|
| 113 |
+
networkx==3.4.2
|
| 114 |
+
fastparquet==2024.11.0
|
| 115 |
+
pure_eval==0.2.3
|
| 116 |
+
opencv-python-headless==4.10.0.84
|
| 117 |
+
scipy==1.15.3
|
| 118 |
+
websocket-client==1.8.0
|
| 119 |
+
nvidia-cusparselt-cu12==0.7.1
|
| 120 |
+
sympy==1.14.0
|
| 121 |
+
gymnasium==0.29.1
|
| 122 |
+
pyarrow==25.0.0
|
| 123 |
+
antlr4-python3-runtime==4.9.3
|
| 124 |
+
pygame==2.6.1
|
| 125 |
+
prompt_toolkit==3.0.53
|
| 126 |
+
pre_commit==4.6.1
|
| 127 |
+
dill==0.4.1
|
| 128 |
+
python-dateutil==2.9.0.post0
|
| 129 |
+
anyio==4.14.2
|
| 130 |
+
yacs==0.1.8
|
| 131 |
+
mergedeep==1.3.4
|
| 132 |
+
ninja==1.13.0
|
| 133 |
+
jinxed==2.1.0
|
| 134 |
+
greenlet==3.5.4
|
| 135 |
+
xxhash==3.8.1
|
| 136 |
+
safetensors==0.8.0
|
| 137 |
+
distlib==0.4.3
|
| 138 |
+
albucore==0.0.17
|
| 139 |
+
multiprocess==0.70.19
|
| 140 |
+
nvidia-nvshmem-cu12==3.4.5
|
| 141 |
+
nvidia-cusolver-cu12==11.7.3.90
|
| 142 |
+
exceptiongroup==1.3.1
|
| 143 |
+
protobuf==7.35.1
|
| 144 |
+
fonttools==4.63.0
|
| 145 |
+
cramjam==2.11.0
|
| 146 |
+
tqdm==4.67.3
|
| 147 |
+
scikit-image==0.25.2
|
| 148 |
+
Pygments==2.20.0
|
| 149 |
+
Markdown==3.10.2
|
| 150 |
+
mypy_extensions==1.1.0
|
| 151 |
+
gpustat==1.1.1
|
| 152 |
+
wcwidth==0.8.2
|
| 153 |
+
GitPython==3.1.57
|
| 154 |
+
Farama-Notifications==0.0.6
|
| 155 |
+
iopath==0.1.10
|
| 156 |
+
accumulation_tree==0.6.4
|
| 157 |
+
zipp==4.1.0
|
| 158 |
+
httpx==0.28.1
|
| 159 |
+
typing-inspect==0.9.0
|
| 160 |
+
peft==0.17.1
|
| 161 |
+
numpydantic==1.6.9
|
| 162 |
+
cfgv==3.5.0
|
| 163 |
+
frozenlist==1.8.0
|
| 164 |
+
hjson==3.1.0
|
| 165 |
+
toml==0.10.2
|
| 166 |
+
Shimmy==0.2.1
|
| 167 |
+
platformdirs==4.11.0
|
| 168 |
+
packaging==26.2
|
| 169 |
+
docstring_parser==0.18.0
|
| 170 |
+
Jinja2==3.1.6
|
| 171 |
+
colorlog==6.12.0
|
| 172 |
+
PyYAML==6.0.3
|
| 173 |
+
pytz==2026.3.post1
|
| 174 |
+
pyyaml-include==1.4.1
|
| 175 |
+
pexpect==4.9.0
|
| 176 |
+
click==8.4.2
|
| 177 |
+
stable_baselines3==2.8.0
|
| 178 |
+
regex==2026.7.19
|
| 179 |
+
yarl==1.24.5
|
| 180 |
+
nvidia-cuda-runtime-cu12==12.8.90
|
| 181 |
+
jedi==0.20.0
|
| 182 |
+
latency-sensitive-bench==0.1.0
|
| 183 |
+
blessed==1.47.0
|
| 184 |
+
nodeenv==1.10.0
|
| 185 |
+
nvidia-nvjitlink-cu12==12.8.93
|
| 186 |
+
gitdb==4.0.12
|
| 187 |
+
albumentations==1.4.18
|
| 188 |
+
deepspeed==0.16.9
|
| 189 |
+
propcache==0.5.2
|
| 190 |
+
websockets==16.1.1
|
| 191 |
+
uv==0.11.32
|
| 192 |
+
fsspec==2026.2.0
|
| 193 |
+
stack-data==0.6.3
|
| 194 |
+
zope.event==6.2
|
| 195 |
+
tifffile==2025.5.10
|
| 196 |
+
multidict==6.7.1
|
| 197 |
+
certifi==2026.7.22
|
| 198 |
+
parso==0.8.7
|
| 199 |
+
nvidia-ml-py==13.610.43
|
| 200 |
+
six==1.17.0
|
| 201 |
+
absl-py==2.5.0
|
| 202 |
+
AutoROM==0.6.1
|
| 203 |
+
python-discovery==1.5.0
|
| 204 |
+
decorator==5.3.1
|
| 205 |
+
virtualenv==21.7.0
|
| 206 |
+
triton==3.6.0
|
| 207 |
+
urllib3==2.7.0
|
| 208 |
+
py-cpuinfo==9.0.0
|
| 209 |
+
pyglet==2.1.15
|
| 210 |
+
signal-slot-mp==1.0.5
|
| 211 |
+
tensorboardX==2.6.5
|
| 212 |
+
async-timeout==5.0.1
|
| 213 |
+
termcolor==3.3.0
|
| 214 |
+
pyparsing==3.3.2
|
| 215 |
+
eva-decord==0.6.1
|
| 216 |
+
eval_type_backport==0.4.0
|
| 217 |
+
tdigest==0.5.2.2
|
| 218 |
+
MarkupSafe==3.0.3
|
| 219 |
+
transformers==4.57.0
|
| 220 |
+
pytest==9.1.1
|
| 221 |
+
sample-factory==2.1.1
|
| 222 |
+
jaraco.functools==4.0.1
|
| 223 |
+
tomli==2.0.1
|
| 224 |
+
importlib_metadata==8.0.0
|
| 225 |
+
packaging==24.2
|
| 226 |
+
jaraco.text==3.12.1
|
| 227 |
+
autocommand==2.2.2
|
| 228 |
+
typeguard==4.3.0
|
| 229 |
+
jaraco.collections==5.1.0
|
| 230 |
+
wheel==0.45.1
|
| 231 |
+
backports.tarfile==1.2.0
|
| 232 |
+
more-itertools==10.3.0
|
| 233 |
+
zipp==3.19.2
|
| 234 |
+
jaraco.context==5.3.0
|
| 235 |
+
platformdirs==4.2.2
|
| 236 |
+
typing_extensions==4.12.2
|
| 237 |
+
inflect==7.3.1
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-1063-nvidia-x86_64-with-glibc2.35",
|
| 3 |
+
"python": "CPython 3.10.20",
|
| 4 |
+
"startedAt": "2026-07-28T06:45:56.290970Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"--config-name",
|
| 7 |
+
"train",
|
| 8 |
+
"model=openvla",
|
| 9 |
+
"env=flappy",
|
| 10 |
+
"init=bridge",
|
| 11 |
+
"mode=single",
|
| 12 |
+
"++framework.qwenvl.base_vlm=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 13 |
+
"++framework.qwenvl.attn_implementation=flash_attention_2",
|
| 14 |
+
"++framework.qwenvl.enable_gradient_checkpointing=true",
|
| 15 |
+
"++framework.action_model.state_dim=7",
|
| 16 |
+
"++framework.action_model.loss_type=discrete_ce",
|
| 17 |
+
"++framework.action_model.action_horizon=1",
|
| 18 |
+
"++framework.action_model.future_action_window_size=0",
|
| 19 |
+
"++framework.action_model.past_action_window_size=0",
|
| 20 |
+
"++framework.action_model.action_dim=7",
|
| 21 |
+
"++framework.action_model.action_env_dim=2",
|
| 22 |
+
"++framework.kv_memory.enabled=false",
|
| 23 |
+
"++framework.kv_memory.window=4",
|
| 24 |
+
"++framework.kv_memory.rollout_len=8",
|
| 25 |
+
"++framework.kv_memory.packed_train=false",
|
| 26 |
+
"++framework.kv_memory.rebased_sink=true",
|
| 27 |
+
"++framework.name=QwenOFT",
|
| 28 |
+
"++datasets.vla_data.dataset_py=lerobot_datasets",
|
| 29 |
+
"++datasets.vla_data.include_state=true",
|
| 30 |
+
"++datasets.vla_data.data_root_dir=playground/Datasets/rl_games",
|
| 31 |
+
"++datasets.vla_data.data_mix=flappy_train",
|
| 32 |
+
"++datasets.vla_data.eval_data_mix=null",
|
| 33 |
+
"++datasets.vla_data.custom_mixtures_path=null",
|
| 34 |
+
"++datasets.vla_data.action_type=discrete",
|
| 35 |
+
"++datasets.vla_data.sequential_step_sampling=false",
|
| 36 |
+
"++datasets.vla_data.eval_sequential_step_sampling=null",
|
| 37 |
+
"++datasets.vla_data.num_workers=8",
|
| 38 |
+
"++datasets.vla_data.eval_num_workers=8",
|
| 39 |
+
"++datasets.vla_data.prefetch_factor=4",
|
| 40 |
+
"++datasets.vla_data.persistent_workers=true",
|
| 41 |
+
"++datasets.vla_data.pin_memory=true",
|
| 42 |
+
"++datasets.vla_data.shuffle=true",
|
| 43 |
+
"++datasets.vla_data.action_balance.enabled=false",
|
| 44 |
+
"++datasets.vla_data.action_balance.strategy=balanced_epoch",
|
| 45 |
+
"++datasets.vla_data.action_balance.action_key=action_id",
|
| 46 |
+
"++datasets.vla_data.action_balance.target_flap_fraction=0.3",
|
| 47 |
+
"++datasets.vla_data.action_balance.noop_id=0",
|
| 48 |
+
"++datasets.vla_data.action_balance.flap_id=1",
|
| 49 |
+
"++datasets.vla_data.latency_curriculum.enabled=false",
|
| 50 |
+
"++datasets.vla_data.latency_curriculum.strategy=exclusive",
|
| 51 |
+
"++datasets.vla_data.latency_curriculum.latencies=null",
|
| 52 |
+
"++datasets.vla_data.latency_curriculum.phase_steps=null",
|
| 53 |
+
"++datasets.vla_data.latency_curriculum.phase_distributions=null",
|
| 54 |
+
"++datasets.vla_data.latency_curriculum.new_latency_passes=1.0",
|
| 55 |
+
"++datasets.vla_data.latency_curriculum.replay_passes=0.25",
|
| 56 |
+
"++datasets.vla_data.latency_curriculum.target_total_passes=2.0",
|
| 57 |
+
"++datasets.vla_data.latency_curriculum.final_equalization=true",
|
| 58 |
+
"++datasets.vla_data.latency_curriculum.step_budget_mode=auto",
|
| 59 |
+
"++datasets.vla_data.latency_curriculum.eval_at_phase_end=false",
|
| 60 |
+
"++datasets.vla_data.latency_curriculum.save_at_phase_end=false",
|
| 61 |
+
"++datasets.vla_data.latency_curriculum.computed_plan=null",
|
| 62 |
+
"++datasets.vla_data.per_device_batch_size=8",
|
| 63 |
+
"++datasets.vla_data.load_all_data_for_training=true",
|
| 64 |
+
"++datasets.vla_data.num_obs_frames=1",
|
| 65 |
+
"++datasets.vla_data.image_mode=single",
|
| 66 |
+
"++datasets.vla_data.prompt_mode=raw",
|
| 67 |
+
"++datasets.vla_data.stitch_grid=[2,2]",
|
| 68 |
+
"++datasets.vla_data.obs_image_size=null",
|
| 69 |
+
"++datasets.vla_data.video_backend=torchvision_av",
|
| 70 |
+
"++dataset.source_hf=",
|
| 71 |
+
"++dataset.config_name=null",
|
| 72 |
+
"++dataset.source_subdir=null",
|
| 73 |
+
"++dataset.converted_name=flappy_train",
|
| 74 |
+
"++dataset.single_source_hf=",
|
| 75 |
+
"++dataset.mixed_source_hf=",
|
| 76 |
+
"++dataset.single_converted_name=flappy_train",
|
| 77 |
+
"++dataset.mixed_converted_name=flappy_mixed_latency_train",
|
| 78 |
+
"++dataset.single_latency_filter=null",
|
| 79 |
+
"++dataset.mixed_latency_filter=null",
|
| 80 |
+
"++dataset.force_download=false",
|
| 81 |
+
"++dataset.setup_force=false",
|
| 82 |
+
"++dataset.skip_verification=false",
|
| 83 |
+
"++dataset.target_latency_unit=raw_frames",
|
| 84 |
+
"++dataset.verify_rows=200",
|
| 85 |
+
"++dataset.max_episodes=null",
|
| 86 |
+
"++dataset.episodes_per_latency=null",
|
| 87 |
+
"++dataset.latency_filter=null",
|
| 88 |
+
"++dataset.debug_subset.enabled=false",
|
| 89 |
+
"++dataset.debug_subset.max_episodes=5",
|
| 90 |
+
"++dataset.debug_subset.suffix=debug",
|
| 91 |
+
"++base_model.repo_id=Qwen/Qwen3-VL-4B-Instruct",
|
| 92 |
+
"++initialization.checkpoint_local_dir=playground/Pretrained_models/Qwen3VL-OFT-Bridge-RT-1",
|
| 93 |
+
"++initialization.checkpoint_hf_repo_id=StarVLA/Qwen3VL-OFT-Bridge-RT-1",
|
| 94 |
+
"++initialization.checkpoint_filename=checkpoints/steps_5000_pytorch_model.pt",
|
| 95 |
+
"++trainer.max_train_steps=4000",
|
| 96 |
+
"++trainer.num_warmup_steps=100",
|
| 97 |
+
"++trainer.save_interval=500",
|
| 98 |
+
"++trainer.eval_interval=250",
|
| 99 |
+
"++trainer.eval_num_batches=50",
|
| 100 |
+
"++trainer.per_latency_eval_num_batches=null",
|
| 101 |
+
"++trainer.eval_action_classification=false",
|
| 102 |
+
"++trainer.eval_action_classification_interval=null",
|
| 103 |
+
"++trainer.cc_f1_tolerance=1",
|
| 104 |
+
"++trainer.learning_rate.base=2e-05",
|
| 105 |
+
"++trainer.learning_rate.qwen_vl_interface=1e-05",
|
| 106 |
+
"++trainer.learning_rate.action_model=0.0001",
|
| 107 |
+
"++trainer.lr_scheduler_type=cosine_with_min_lr",
|
| 108 |
+
"++trainer.scheduler_specific_kwargs.min_lr=1e-06",
|
| 109 |
+
"++trainer.freeze_modules=",
|
| 110 |
+
"++trainer.freeze_vit=false",
|
| 111 |
+
"++trainer.freeze_tied_embedding=false",
|
| 112 |
+
"++trainer.freeze_llm_layers=[]",
|
| 113 |
+
"++trainer.loss_scale.vla=1.0",
|
| 114 |
+
"++trainer.loss_scale.vlm=0.1",
|
| 115 |
+
"++trainer.max_grad_norm=1.0",
|
| 116 |
+
"++trainer.weight_decay=0.0",
|
| 117 |
+
"++trainer.logging_frequency=1",
|
| 118 |
+
"++trainer.profile_timing.enabled=false",
|
| 119 |
+
"++trainer.profile_timing.log_interval=10",
|
| 120 |
+
"++trainer.gradient_clipping=1.0",
|
| 121 |
+
"++trainer.gradient_accumulation_steps=16",
|
| 122 |
+
"++trainer.distributed_backend=deepspeed",
|
| 123 |
+
"++trainer.is_resume=false",
|
| 124 |
+
"++trainer.pretrained_checkpoint=null",
|
| 125 |
+
"++trainer.resume_step=0",
|
| 126 |
+
"++trainer.reload_modules=null",
|
| 127 |
+
"++trainer.optimizer.name=AdamW",
|
| 128 |
+
"++trainer.optimizer.betas=[0.9,0.95]",
|
| 129 |
+
"++trainer.optimizer.eps=1e-08",
|
| 130 |
+
"++trainer.optimizer.weight_decay=1e-08",
|
| 131 |
+
"++trainer.optimizer.fused=true",
|
| 132 |
+
"++trainer.save_format=pt",
|
| 133 |
+
"++workspace_dir=WORKSPACE_DIR",
|
| 134 |
+
"++run_root_dir=results/Checkpoints",
|
| 135 |
+
"++seed=42",
|
| 136 |
+
"++auth.env_file=null",
|
| 137 |
+
"++auth.hf_token_env=HF_TOKEN",
|
| 138 |
+
"++auth.wandb_api_key_env=WANDB_API_KEY",
|
| 139 |
+
"++paths.run_root_dir=results/Checkpoints",
|
| 140 |
+
"++paths.dataset_local_dir=data/memory/flappy_fix_latency_3_200ep_7k2steps",
|
| 141 |
+
"++paths.dataset_cache_dir=null",
|
| 142 |
+
"++paths.base_model_dir=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 143 |
+
"++paths.accelerate_config=starVLA/config/deepseeds/deepspeed_zero2.yaml",
|
| 144 |
+
"++rl_games.model_alias=openvla",
|
| 145 |
+
"++rl_games.env_eval.image_size=224",
|
| 146 |
+
"++rl_games.env_eval.frameskip=1",
|
| 147 |
+
"++rl_games.env_eval.image_transform=raw_rgb",
|
| 148 |
+
"++rl_games.env_eval.prompt_mode=raw",
|
| 149 |
+
"++rl_games.env_eval.ghost_trail.history_frames=5",
|
| 150 |
+
"++rl_games.env_eval.ghost_trail.gamma=1.3",
|
| 151 |
+
"++rl_games.env_eval.ghost_trail.min_alpha=35",
|
| 152 |
+
"++rl_games.env_eval.ghost_trail.scroll_px_per_step=4.0",
|
| 153 |
+
"++rl_games.env_eval.ghost_trail.ground_fraction=0.22",
|
| 154 |
+
"++rl_games.env_eval.seed=42",
|
| 155 |
+
"++rl_games.env_eval.fixed_episode_seeds=true",
|
| 156 |
+
"++rl_games.env_eval.latency_seed_stride=0",
|
| 157 |
+
"++rl_games.env_eval.task_seed_stride=0",
|
| 158 |
+
"++rl_games.env_eval.task_description=",
|
| 159 |
+
"++rl_games.env_eval.eval_parallel_envs=5",
|
| 160 |
+
"++rl_games.env_eval.action_chunk_execution.enabled=false",
|
| 161 |
+
"++rl_games.env_eval.action_chunk_execution.chunk_size=null",
|
| 162 |
+
"++rl_games.env_eval.enabled=true",
|
| 163 |
+
"++rl_games.env_eval.eval_backend=latency_bench",
|
| 164 |
+
"++rl_games.env_eval.distributed_mode=rank_sharded",
|
| 165 |
+
"++rl_games.env_eval.vectorized.enabled=false",
|
| 166 |
+
"++rl_games.env_eval.vectorized.batch_size=1",
|
| 167 |
+
"++rl_games.env_eval.latency.prompt_map_path=null",
|
| 168 |
+
"++rl_games.env_eval.latency.mode=single",
|
| 169 |
+
"++rl_games.env_eval.latency.values=[0]",
|
| 170 |
+
"++rl_games.env_eval.mid_train.enabled=true",
|
| 171 |
+
"++rl_games.env_eval.mid_train.interval_steps=250",
|
| 172 |
+
"++rl_games.env_eval.mid_train.latencies=[3]",
|
| 173 |
+
"++rl_games.env_eval.mid_train.num_episodes=20",
|
| 174 |
+
"++rl_games.env_eval.mid_train.max_steps_per_episode=3600",
|
| 175 |
+
"++rl_games.env_eval.post_train.enabled=true",
|
| 176 |
+
"++rl_games.env_eval.post_train.latencies=[3]",
|
| 177 |
+
"++rl_games.env_eval.post_train.num_episodes=50",
|
| 178 |
+
"++rl_games.env_eval.post_train.max_steps_per_episode=3600",
|
| 179 |
+
"++rl_games.task=flappy",
|
| 180 |
+
"++rl_games.initialization_mode=bridge",
|
| 181 |
+
"++rl_games.action_carrier=bridge",
|
| 182 |
+
"++bridge_base_model.repo_id.openvla=Qwen/Qwen3-VL-4B-Instruct",
|
| 183 |
+
"++bridge_base_model.repo_id.pi0=StarVLA/Qwen2.5-VL-3B-Instruct-Action",
|
| 184 |
+
"++bridge_base_model.repo_id.pi05=Qwen/Qwen3-VL-4B-Instruct",
|
| 185 |
+
"++bridge_base_model.repo_id.gr00t=Qwen/Qwen3-VL-4B-Instruct",
|
| 186 |
+
"++bridge_base_model.local_dir.openvla=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 187 |
+
"++bridge_base_model.local_dir.pi0=playground/Pretrained_models/Qwen2.5-VL-3B-Instruct-Action",
|
| 188 |
+
"++bridge_base_model.local_dir.pi05=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 189 |
+
"++bridge_base_model.local_dir.gr00t=playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 190 |
+
"++checkpoint.load=auto",
|
| 191 |
+
"++checkpoint.hf_repo_id=null",
|
| 192 |
+
"++checkpoint.save_best_model=false",
|
| 193 |
+
"++checkpoint.save_final_model=true",
|
| 194 |
+
"++checkpoint.save_pt_file=false",
|
| 195 |
+
"++checkpoint.save_training_state=true",
|
| 196 |
+
"++checkpoint.save_safetensors_file=true",
|
| 197 |
+
"++checkpoint.local.keep_last_n=1",
|
| 198 |
+
"++checkpoint.sync.enabled=false",
|
| 199 |
+
"++checkpoint.sync.repo_id=null",
|
| 200 |
+
"++checkpoint.sync.keep_last_n=0",
|
| 201 |
+
"++checkpoint.sync.sync_every_n_checkpoints=1",
|
| 202 |
+
"++checkpoint.sync.resume_policy=local_latest",
|
| 203 |
+
"++run_id=flappy_fix_latency_3_200ep_7k2steps_single_baseline",
|
| 204 |
+
"++output_dir=null",
|
| 205 |
+
"++config_yaml=null",
|
| 206 |
+
"++is_debug=false",
|
| 207 |
+
"++version_id=0.21",
|
| 208 |
+
"++run_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints",
|
| 209 |
+
"++trainer.is_resume=true",
|
| 210 |
+
"++trainer.pretrained_checkpoint=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/checkpoints/steps_4000_state",
|
| 211 |
+
"++trainer.resume_step=4000",
|
| 212 |
+
"++datasets.vla_data.data_root_dir=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps",
|
| 213 |
+
"++datasets.vla_data.data_mix=flappy_train__bridge",
|
| 214 |
+
"++datasets.vla_data.eval_data_mix=flappy_train__bridge__val",
|
| 215 |
+
"++framework.qwenvl.base_vlm=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/playground/Pretrained_models/Qwen3-VL-4B-Instruct",
|
| 216 |
+
"++rl_games.env_eval.latency.prompt_map_path=/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/data/memory/flappy_fix_latency_3_200ep_7k2steps/flappy_train__bridge/latency_prompt_map.json"
|
| 217 |
+
],
|
| 218 |
+
"program": "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/starVLA/training/train_starvla_hydra.py",
|
| 219 |
+
"codePath": "starVLA/training/train_starvla_hydra.py",
|
| 220 |
+
"codePathLocal": "starVLA/training/train_starvla_hydra.py",
|
| 221 |
+
"git": {
|
| 222 |
+
"remote": "git@github.com:talha1503/starVLA.git",
|
| 223 |
+
"commit": "9662ad113a8910207f7a17b0faab3480c78756f1"
|
| 224 |
+
},
|
| 225 |
+
"email": "zihanwang2029@u.northwestern.edu",
|
| 226 |
+
"root": "/lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb",
|
| 227 |
+
"host": "pool0-01735",
|
| 228 |
+
"executable": "/lustre/fsw/portfolios/nvr/users/zihwang/miniconda3/envs/starvla_rl_games_openvla/bin/python",
|
| 229 |
+
"cpu_count": 64,
|
| 230 |
+
"cpu_count_logical": 128,
|
| 231 |
+
"gpu": "NVIDIA H100 80GB HBM3",
|
| 232 |
+
"gpu_count": 2,
|
| 233 |
+
"disk": {
|
| 234 |
+
"/": {
|
| 235 |
+
"total": "7651200073728",
|
| 236 |
+
"used": "253479514112"
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
"memory": {
|
| 240 |
+
"total": "2164170432512"
|
| 241 |
+
},
|
| 242 |
+
"gpu_nvidia": [
|
| 243 |
+
{
|
| 244 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 245 |
+
"memoryTotal": "85520809984",
|
| 246 |
+
"cudaCores": 16896,
|
| 247 |
+
"architecture": "Hopper",
|
| 248 |
+
"uuid": "GPU-07201b7e-fb4f-2c0f-222e-76eeda5e5c7b"
|
| 249 |
+
},
|
| 250 |
+
{
|
| 251 |
+
"name": "NVIDIA H100 80GB HBM3",
|
| 252 |
+
"memoryTotal": "85520809984",
|
| 253 |
+
"cudaCores": 16896,
|
| 254 |
+
"architecture": "Hopper",
|
| 255 |
+
"uuid": "GPU-6be700cf-9e97-8860-a313-d824469798ba"
|
| 256 |
+
}
|
| 257 |
+
],
|
| 258 |
+
"cudaVersion": "12.2",
|
| 259 |
+
"slurm": {
|
| 260 |
+
"array_job_id": "14462237",
|
| 261 |
+
"array_task_count": "1",
|
| 262 |
+
"array_task_id": "0",
|
| 263 |
+
"array_task_max": "0",
|
| 264 |
+
"array_task_min": "0",
|
| 265 |
+
"array_task_step": "1",
|
| 266 |
+
"cluster_name": "cw-dfw-cs-001",
|
| 267 |
+
"conf": "/cm/shared/apps/slurm/var/etc/cw-dfw-cs-001/slurm.conf",
|
| 268 |
+
"cpus_on_node": "64",
|
| 269 |
+
"cpus_per_task": "64",
|
| 270 |
+
"gpus_on_node": "2",
|
| 271 |
+
"gtids": "0",
|
| 272 |
+
"job_account": "nvr_lacr_llm",
|
| 273 |
+
"job_cpus_per_node": "64",
|
| 274 |
+
"job_end_time": "1785235406",
|
| 275 |
+
"job_gid": "30",
|
| 276 |
+
"job_gpus": "5,6",
|
| 277 |
+
"job_id": "14462237",
|
| 278 |
+
"job_name": "mem-train",
|
| 279 |
+
"job_nodelist": "pool0-01735",
|
| 280 |
+
"job_num_nodes": "1",
|
| 281 |
+
"job_partition": "batch",
|
| 282 |
+
"job_qos": "normal",
|
| 283 |
+
"job_start_time": "1785221006",
|
| 284 |
+
"job_uid": "159489",
|
| 285 |
+
"job_user": "zihwang",
|
| 286 |
+
"jobid": "14462237",
|
| 287 |
+
"localid": "0",
|
| 288 |
+
"mem_per_node": "524288",
|
| 289 |
+
"nnodes": "1",
|
| 290 |
+
"nodeid": "0",
|
| 291 |
+
"nodelist": "pool0-01735",
|
| 292 |
+
"nprocs": "1",
|
| 293 |
+
"ntasks": "1",
|
| 294 |
+
"prio_process": "0",
|
| 295 |
+
"procid": "0",
|
| 296 |
+
"submit_dir": "/home/zihwang/projects",
|
| 297 |
+
"submit_host": "cw-dfw-cs-001-vscode-01",
|
| 298 |
+
"task_pid": "310414",
|
| 299 |
+
"tasks_per_node": "1",
|
| 300 |
+
"topology_addr": "C1.S1.L207-DH4.pool0-01735",
|
| 301 |
+
"topology_addr_pattern": "switch.switch.switch.node",
|
| 302 |
+
"tres_per_task": "cpu=64"
|
| 303 |
+
},
|
| 304 |
+
"writerId": "q0u7jykkbxijm93bjntw1djtrjrend2j"
|
| 305 |
+
}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"rl_games_eval/post_train/mean_reward":2.0000000447034836,"rl_games_eval/post_train/mean_length":31,"rl_games_eval/post_train/macro_mean_length":31,"rl_games_eval/post_train/task_count":1,"rl_games_eval/post_train/flappy__latency_3/mean_length":31,"_wandb":{"runtime":68},"rl_games_eval/post_train/macro_mean_reward":2.0000000447034836,"rl_games_eval/post_train/flappy__latency_3/mean_reward":2.0000000447034836,"rl_games_eval/post_train/std_reward":0,"rl_games_eval/post_train/flappy__latency_3/num_episodes":50,"rl_games_eval/post_train/flappy__latency_3/std_reward":0,"_runtime":68.330780643,"_timestamp":1.7852212260588396e+09,"_step":4000,"rl_games_eval/post_train/std_length":0,"rl_games_eval/post_train/flappy__latency_3/std_length":0,"rl_games_eval/post_train/total_episodes":50}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug-core.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-27T23:45:56.839105029-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpyn9ooqij/port-311399.txt","pid":311399,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-07-27T23:45:56.840322841-07:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":311399}
|
| 3 |
+
{"time":"2026-07-27T23:45:56.840322693-07:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-311399-312425-1646591483/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-07-27T23:45:56.992573843-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-07-27T23:45:57.005467716-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"m3s24v9k","id":"1(@)"}
|
| 6 |
+
{"time":"2026-07-27T23:45:57.383236116-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"m3s24v9k","id":"1(@)"}
|
| 7 |
+
{"time":"2026-07-27T23:46:03.121462751-07:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rhbeklxc69u9"}
|
| 8 |
+
{"time":"2026-07-27T23:47:06.059953753-07:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rhbeklxc69u9"}
|
| 9 |
+
{"time":"2026-07-27T23:47:07.626009229-07:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rhbeklxc69u9"}
|
| 10 |
+
{"time":"2026-07-27T23:47:07.627028305-07:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"m3s24v9k","id":"1(@)"}
|
| 11 |
+
{"time":"2026-07-27T23:47:07.628946835-07:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"m3s24v9k","id":"1(@)"}
|
| 12 |
+
{"time":"2026-07-27T23:47:10.372437695-07:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 13 |
+
{"time":"2026-07-27T23:47:10.372462032-07:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-07-27T23:47:10.372482376-07:00","level":"INFO","msg":"server is shutting down"}
|
| 15 |
+
{"time":"2026-07-27T23:47:10.372477407-07:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 16 |
+
{"time":"2026-07-27T23:47:10.372532166-07:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 17 |
+
{"time":"2026-07-27T23:47:10.372534496-07:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 18 |
+
{"time":"2026-07-27T23:47:10.37257131-07:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-311399-312425-1646591483/socket","Net":"unix"}}
|
| 19 |
+
{"time":"2026-07-27T23:47:10.372606299-07:00","level":"INFO","msg":"server is closed"}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-27T23:45:57.00634205-07:00","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-07-27T23:45:57.008719471-07:00","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
|
| 3 |
+
{"time":"2026-07-27T23:45:57.382134905-07:00","level":"INFO","msg":"stream: created new stream","id":"m3s24v9k"}
|
| 4 |
+
{"time":"2026-07-27T23:45:57.382222854-07:00","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-07-27T23:45:57.383221752-07:00","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-07-27T23:45:57.383234242-07:00","level":"INFO","msg":"writer: started","stream_id":"m3s24v9k"}
|
| 7 |
+
{"time":"2026-07-27T23:45:57.383248472-07:00","level":"INFO","msg":"sender: started"}
|
| 8 |
+
{"time":"2026-07-27T23:45:58.126453839-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-07-27T23:45:58.505064782-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-07-27T23:46:13.127485949-07:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":8,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-07-27T23:46:13.484399712-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-07-27T23:46:28.126991591-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
|
| 13 |
+
{"time":"2026-07-27T23:46:28.525664421-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-07-27T23:46:43.126641091-07:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
|
| 15 |
+
{"time":"2026-07-27T23:46:43.429954528-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-07-27T23:46:58.1272039-07:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":9,"console_lines":5}
|
| 17 |
+
{"time":"2026-07-27T23:46:58.483867647-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-07-27T23:47:07.231562723-07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 19 |
+
{"time":"2026-07-27T23:47:07.231696217-07:00","level":"INFO","msg":"filestream: sending request","total_files":3,"history_offset":0,"history_lines":1,"console_offset":14,"console_lines":22,"uploaded_len":3,"complete":true,"exit_code":0}
|
| 20 |
+
{"time":"2026-07-27T23:47:07.623031711-07:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 21 |
+
{"time":"2026-07-27T23:47:07.62379316-07:00","level":"INFO","msg":"handler: operation stats","stats":{}}
|
| 22 |
+
{"time":"2026-07-27T23:47:07.627040817-07:00","level":"INFO","msg":"stream: finishing up"}
|
| 23 |
+
{"time":"2026-07-27T23:47:07.627093121-07:00","level":"INFO","msg":"handler: closed"}
|
| 24 |
+
{"time":"2026-07-27T23:47:07.627789421-07:00","level":"INFO","msg":"sender: closed"}
|
| 25 |
+
{"time":"2026-07-27T23:47:07.627798284-07:00","level":"INFO","msg":"stream: all finished"}
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug.log
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-27 23:45:56,299 INFO MainThread:311399 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
|
| 2 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_setup.py:_flush():81] Configure stats pid to 311399
|
| 3 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug.log
|
| 5 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /lustre/fsw/portfolios/nvr/projects/nvr_lacr_llm/users/zihwang/latency-sensitive-bench/third_party/starVLA/results/Checkpoints/flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/logs/debug-internal.log
|
| 6 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:init():850] calling init triggers
|
| 7 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'_wandb': {}}
|
| 9 |
+
2026-07-27 23:45:56,300 INFO MainThread:311399 [wandb_init.py:init():898] starting backend
|
| 10 |
+
2026-07-27 23:45:56,992 INFO MainThread:311399 [wandb_init.py:init():913] sending inform_init request
|
| 11 |
+
2026-07-27 23:45:57,383 INFO MainThread:311399 [wandb_init.py:init():918] backend started and connected
|
| 12 |
+
2026-07-27 23:45:57,385 INFO MainThread:311399 [wandb_init.py:init():988] updated telemetry
|
| 13 |
+
2026-07-27 23:45:57,415 INFO MainThread:311399 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-27 23:45:57,729 INFO MainThread:311399 [wandb_init.py:init():1056] starting run threads in backend
|
| 15 |
+
2026-07-27 23:45:58,115 INFO MainThread:311399 [wandb_run.py:_console_start():2554] atexit reg
|
| 16 |
+
2026-07-27 23:45:58,115 INFO MainThread:311399 [wandb_run.py:_redirect():2403] redirect: wrap_raw
|
| 17 |
+
2026-07-27 23:45:58,115 INFO MainThread:311399 [wandb_run.py:_redirect():2472] Wrapping output streams.
|
| 18 |
+
2026-07-27 23:45:58,115 INFO MainThread:311399 [wandb_run.py:_redirect():2495] Redirects installed.
|
| 19 |
+
2026-07-27 23:45:58,125 INFO MainThread:311399 [wandb_init.py:init():1094] run started, returning control to user process
|
| 20 |
+
2026-07-27 23:47:06,059 INFO MainThread:311399 [wandb_run.py:_finish():2321] finishing run zihanwang-ai-northwestern-university/starVLA_rl_games/m3s24v9k
|
| 21 |
+
2026-07-27 23:47:06,059 INFO MainThread:311399 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
|
| 22 |
+
2026-07-27 23:47:06,059 INFO MainThread:311399 [wandb_run.py:_restore():2502] restore
|
| 23 |
+
2026-07-27 23:47:06,060 INFO MainThread:311399 [wandb_run.py:_restore():2508] restore done
|
| 24 |
+
2026-07-27 23:47:07,626 INFO MainThread:311399 [wandb_run.py:_footer_sync_info():3889] logging synced files
|
flappy_fix_latency_3_200ep_7k2steps_single_baseline/wandb/wandb/run-20260727_234556-m3s24v9k/run-m3s24v9k.wandb
ADDED
|
Binary file (28.2 kB). View file
|
|
|