add lafm_vla_flow_neo2b_fulllayer_libero_fullft_m323_flft0815r/config.yaml

lafm_vla_flow_neo2b_fulllayer_libero_fullft_m323_flft0815r/config.yaml

@@ -0,0 +1,243 @@
+output_dir: /mnt/afs/multimodal_data/fanweichen/experiments
+train_split: 0.9
+img_size: 224
+patch_size: 14
+encoder_scale: tiny
+dump_object: true
+num_workers: 8
+seed: 42
+log_every: 20
+trainer:
+  max_epochs: 1
+  max_steps: 30000
+  devices: auto
+  accelerator: gpu
+  strategy: ddp
+  precision: bf16
+  gradient_accumulation_steps: 4
+  gradient_clip_val: 1.0
+  save_every: 2500
+  save_final: true
+  overfit_first_batch: false
+  num_sanity_val_steps: 0
+  sync_batchnorm: false
+loader:
+  batch_size: 8
+  num_workers: 8
+  persistent_workers: true
+  prefetch_factor: 4
+  pin_memory: true
+optimizer:
+  type: AdamW
+  lr: ${learning_rate}
+  weight_decay: ${weight_decay}
+wandb:
+  enabled: false
+  config:
+    entity: weepies
+    project: lewm
+    name: ${output_model_name}
+    id: ${subdir}
+    resume: allow
+    log_model: false
+wm:
+  history_size: 1
+  num_preds: 8
+  embed_dim: 2048
+  type: lafm_vla_flow
+  action_dim: 7
+  proprio_dim: 8
+data:
+  train:
+    _target_: data.lerobot.robot_video_dataset.RobotVideoDataset
+    dataset_dirs:
+    - downloads/FastWAM/data/libero_m323_rs141_256/libero_spatial_no_noops_lerobot
+    - downloads/FastWAM/data/libero_m323_rs141_256/libero_object_no_noops_lerobot
+    - downloads/FastWAM/data/libero_m323_rs141_256/libero_goal_no_noops_lerobot
+    - downloads/FastWAM/data/libero_m323_rs141_256/libero_10_no_noops_lerobot
+    shape_meta:
+      images:
+      - key: image
+        raw_shape:
+        - 3
+        - 256
+        - 256
+        shape:
+        - 3
+        - 256
+        - 256
+      - key: wrist_image
+        raw_shape:
+        - 3
+        - 256
+        - 256
+        shape:
+        - 3
+        - 256
+        - 256
+      action:
+      - key: default
+        raw_shape: 7
+        shape: 7
+      state:
+      - key: default
+        raw_shape: 8
+        shape: 8
+    num_frames: 9
+    global_sample_stride: 1
+    action_video_freq_ratio: 2
+    video_size:
+    - 256
+    - 512
+    camera_key: null
+    val_set_proportion: 0.0
+    is_training_set: true
+    skip_padding_as_possible: true
+    concat_multi_camera: horizontal
+    image_current_frame_only: true
+    fast_arrow_read: true
+    prewarm_page_cache: true
+    processor:
+      _target_: data.lerobot.processors.fastwam_processor.FastWAMProcessor
+      shape_meta: ${data.train.shape_meta}
+      num_obs_steps: ${data.train.num_frames}
+      image_obs_steps: 1
+      num_output_cameras: 2
+      action_output_dim: 7
+      proprio_output_dim: 8
+      delta_action_dim_mask:
+        default:
+        - true
+        - true
+        - true
+        - true
+        - true
+        - true
+        - false
+      action_state_transforms: null
+      use_stepwise_action_norm: false
+      norm_default_mode: -1/1
+      norm_exception_mode: null
+      action_state_merger:
+        _target_: data.lerobot.transforms.action_state_merger.ConcatLeftAlign
+      train_transforms:
+      - _target_: data.lerobot.transforms.image.ToTensor
+      - _target_: torchvision.transforms.Resize
+        size:
+        - 256
+        - 256
+      val_transforms:
+      - _target_: data.lerobot.transforms.image.ToTensor
+      - _target_: torchvision.transforms.Resize
+        size:
+        - 256
+        - 256
+    text_embedding_cache_dir: ${oc.env:LAFM_LIBERO_TEXT_CACHE,downloads/FastWAM/data/text_embeds_cache/libero}
+    context_len: 128
+    pretrained_norm_stats: ${oc.env:LAFM_OXE_NORM_STATS,downloads/openvla_oxe_lerobot/dataset_stats.json}
+    per_dataset_action_norm: true
+    per_dataset_norm_stats_dir: downloads/FastWAM/data/libero_m323_rs141_256/_per_dataset_stats_merged
+    action_norm_mask:
+    - true
+    - true
+    - true
+    - true
+    - true
+    - true
+    - false
+    dataset_weight_sampling: true
+    dataset_weights:
+      libero_spatial_no_noops_lerobot: 1.0
+      libero_object_no_noops_lerobot: 1.0
+      libero_goal_no_noops_lerobot: 1.0
+      libero_10_no_noops_lerobot: 1.0
+method: lafm_flow_fullft_skip_mem
+output_model_name: lafm_vla_flow_neo2b_fulllayer_libero_fullft_m323
+subdir: lafm_vla_flow_neo2b_fulllayer_libero_fullft_m323_flft0815r
+save_note: 'LIBERO 4-suite FULL finetune of fulllayer-bb (FULL-LAYER 40/40 layer-map
+  ablation, 4096x75k pretrain @75k) (NEO VLM unfrozen; no grad-dead backbone tail
+  under the 0..39 map, stock last-layer/final-norm freeze only). [m323_rs141_256 data:
+  mujoco 3.2.3 + robosuite 1.4.1 regen, 256-native]'
+mixed_precision: bf16
+learning_rate: 5.0e-05
+weight_decay: 0.0
+batch_size: 8
+model:
+  architecture: lafm_vla_flow
+  backbone_type: neo_frozen
+  neo_model_path: /mnt/afs/diaohaiwen/workspace/Neo/Neo_dense/RUN/NEOpp-2B-SFT
+  local_files_only: true
+  image_size: 256
+  max_text_tokens: 128
+  max_views: 2
+  chunk_K: 8
+  init_from: /mnt/afs/multimodal_data/fanweichen/experiments/lafm_vla_flow_neo2b_fulllayer_bb_bb_flbb_0813a/lafm_vla_flow_neo2b_fulllayer_bb_bb_weights.ckpt.shards.step000075000
+  init_use_ema: true
+  init_full_cover: false
+  aug:
+    enabled: true
+    crop_frac: 0.95
+    rot_deg: 5.0
+    jitter:
+    - 0.3
+    - 0.4
+    - 0.5
+    jitter_p: 0.5
+  expert:
+    width: 1024
+    intermediate: 3072
+    init_std: 0.02
+    qk_norm_calibration: true
+    layer_map:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    - 11
+    - 12
+    - 13
+    - 14
+    - 15
+    - 16
+    - 17
+    - 18
+    - 19
+    - 20
+    - 21
+    - 22
+    - 23
+    - 24
+    - 25
+    - 26
+    - 27
+    - 28
+    - 29
+    - 30
+    - 31
+    - 32
+    - 33
+    - 34
+    - 35
+    - 36
+    - 37
+    - 38
+    - 39
+  flow:
+    num_steps: 10
+  adam_betas:
+  - 0.9
+  - 0.95
+  adam_eps: 1.0e-08
+  adam_weight_decay: 1.0e-10
+  warmup_steps: 2000
+  lr_final: 5.0e-06
+  ema_decay: 0.999
+  eval_every: 4000
+  sample_eval_every: 8000