run_id: 0625_WanMoTMultiViewJoint_flexiv_2views
run_root_dir: ./results/Checkpoints/vla
seed: 42
trackers:
- jsonl
is_debug: false
framework:
  name: WanMoTMultiViewJoint
  torch_dtype: bfloat16
  skip_dit_load_from_pretrain: false
  concat_multi_camera: horizontal
  view_image_size:
  - 224
  - 224
  video_model:
    model_id: Wan-AI/Wan2.2-TI2V-5B
    tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
    tokenizer_max_len: 128
    load_text_encoder: false
    redirect_common_files: true
    dit_path: playground/Pretrained_models/Wan2.2-TI2V-5B
    vae_path: playground/Pretrained_models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
    text_encoder_path: playground/Pretrained_models/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
    tokenizer_path: playground/Pretrained_models/umt5-xxl
    z_dim: 48
    vae_dim: 160
    enable_prompt_cache: true
    prefer_cached_text_context: true
    text_mask_padding_as_valid: true
    prompt_template: 'A video recorded from a robot''s point of view executing the
      following instruction: {task}'
    config:
      has_image_input: false
      patch_size:
      - 1
      - 2
      - 2
      in_dim: 48
      hidden_dim: 3072
      ffn_dim: 14336
      freq_dim: 256
      text_dim: 4096
      out_dim: 48
      num_heads: 24
      attn_head_dim: 128
      num_layers: 30
      eps: 1.0e-06
      seperated_timestep: true
      require_clip_embedding: false
      require_vae_embedding: false
      fuse_vae_embedding_in_latents: true
      action_conditioned: false
      action_group_causal_mask_mode: group_diagonal
      video_attention_mask_mode: first_frame_causal
      use_gradient_checkpointing: ${trainer.enable_gradient_checkpointing}
    scheduler:
      train_shift: 5.0
      infer_shift: 5.0
      num_train_timesteps: 1000
    text_encoder_kwargs:
      dim: 4096
      dim_attn: 4096
      dim_ffn: 10240
      num_heads: 64
      num_layers: 24
      num_buckets: 32
      shared_pos: false
      dropout: 0.1
  action_model:
    action_horizon: 48
    action_dim: 8
    state_dim: 8
    num_inference_steps: 10
    model_path: results/Checkpoints/WanMoT/ActionExpert_linear_interp_Wan22_alphascale_1024hdim.pt
    skip_load_from_pretrain: ${framework.skip_dit_load_from_pretrain}
    supervision:
      target: action_space
      weight_mode: none
    config:
      action_dim: ${framework.action_model.action_dim}
      hidden_dim: 1024
      ffn_dim: 4096
      num_heads: 24
      attn_head_dim: 128
      num_layers: 30
      text_dim: ${framework.video_model.config.text_dim}
      freq_dim: 256
      eps: 1.0e-06
      use_gradient_checkpointing: ${trainer.enable_gradient_checkpointing}
    scheduler:
      train_shift: 5.0
      infer_shift: 5.0
      num_train_timesteps: 1000
      sample_mode: beta_tau
      noise_beta_alpha: 1.5
      noise_beta_beta: 1.0
      noise_s: 0.999
  loss:
    lambda_video: 1.0
    lambda_action: 1.0
datasets:
  vla_data:
    dataset_py: vla_datasets
    data_mix: flexiv_2views
    action_horizon: ${framework.action_model.action_horizon}
    action_pad_size: ${framework.action_model.action_dim}
    state_pad_size: ${framework.action_model.state_dim}
    disable_state: false
    image_size: ${framework.view_image_size}
    use_future_frames: true
    num_future_frames: 12
    future_frame_stride: 4
    use_decord: true
    vlm_obs_delay: 0
    text_embedding_cache_dir: data/text_embeds_cache/flexiv
    require_text_embedding_cache: true
    text_context_len: ${framework.video_model.tokenizer_max_len}
    text_cache_encoder_id: wan22ti2v5b
    text_context_prompt_template: ${framework.video_model.prompt_template}
    per_device_batch_size: 8
    split_strategy: episode_ratio
    eval_ratio: 0.02
    split_seed: 42
    eval_per_device_batch_size: 4
    num_workers: 8
trainer:
  epochs: 100
  max_train_steps: 20000
  num_warmup_steps: 1000
  save_interval: 5000
  eval_interval: 200
  logging_frequency: 10
  learning_rate:
    base: 0.0001
  lr_scheduler_type: cosine_with_min_lr
  scheduler_specific_kwargs:
    min_lr: 1.0e-06
  freeze_modules: null
  max_grad_norm: 1.0
  warmup_ratio: 0.05
  weight_decay: 1.0e-05
  gradient_clipping: 1.0
  gradient_accumulation_steps: 1
  knowledge_isolation: false
  enable_gradient_checkpointing: true
  enable_mixed_precision_training: true
  enable_compile: true
  resume_from_checkpoint: latest
  init_from_stage_checkpoint: null
  optimizer:
    name: AdamW
    betas:
    - 0.9
    - 0.95
    eps: 1.0e-08
    weight_decay: 0.01
output_dir: ./results/Checkpoints/vla/0625_WanMoTMultiViewJoint_flexiv_2views